2024-12-05 07:39:48,576 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 07:39:48,586 main DEBUG Took 0.008636 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 07:39:48,586 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 07:39:48,587 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 07:39:48,588 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 07:39:48,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,596 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 07:39:48,607 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,609 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,610 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,611 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,612 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,612 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,613 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,614 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,614 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,615 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,616 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,616 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,617 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,617 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,618 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,618 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,619 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,619 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,620 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,620 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,620 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,621 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,622 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 07:39:48,622 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,623 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 07:39:48,624 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 07:39:48,626 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 07:39:48,628 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 07:39:48,629 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 07:39:48,630 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 07:39:48,631 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 07:39:48,642 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 07:39:48,645 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 07:39:48,647 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 07:39:48,647 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 07:39:48,648 main DEBUG createAppenders(={Console}) 2024-12-05 07:39:48,649 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-05 07:39:48,650 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 07:39:48,650 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-05 07:39:48,651 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 07:39:48,651 main DEBUG OutputStream closed 2024-12-05 07:39:48,651 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 07:39:48,651 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 07:39:48,652 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-05 07:39:48,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 07:39:48,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 07:39:48,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 07:39:48,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 07:39:48,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 07:39:48,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 07:39:48,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 07:39:48,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 07:39:48,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 07:39:48,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 07:39:48,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 07:39:48,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 07:39:48,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 07:39:48,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 07:39:48,733 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 07:39:48,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 07:39:48,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 07:39:48,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 07:39:48,736 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 07:39:48,737 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-05 07:39:48,737 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 07:39:48,737 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-05T07:39:48,946 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5 2024-12-05 07:39:48,949 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 07:39:48,949 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T07:39:48,957 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-12-05T07:39:48,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T07:39:48,985 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/cluster_8fe0b83e-62de-a3e8-93ab-b871591f10b7, deleteOnExit=true 2024-12-05T07:39:48,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T07:39:48,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/test.cache.data in system properties and HBase conf 2024-12-05T07:39:48,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T07:39:48,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/hadoop.log.dir in system properties and HBase conf 2024-12-05T07:39:48,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T07:39:48,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T07:39:48,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T07:39:49,066 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T07:39:49,179 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T07:39:49,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T07:39:49,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T07:39:49,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T07:39:49,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T07:39:49,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T07:39:49,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T07:39:49,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T07:39:49,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T07:39:49,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T07:39:49,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/nfs.dump.dir in system properties and HBase conf 2024-12-05T07:39:49,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/java.io.tmpdir in system properties and HBase conf 2024-12-05T07:39:49,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T07:39:49,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T07:39:49,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T07:39:50,156 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T07:39:50,269 INFO [Time-limited test {}] log.Log(170): Logging initialized @2318ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T07:39:50,381 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T07:39:50,482 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T07:39:50,509 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T07:39:50,510 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T07:39:50,511 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T07:39:50,525 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T07:39:50,530 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/hadoop.log.dir/,AVAILABLE} 2024-12-05T07:39:50,531 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T07:39:50,761 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/java.io.tmpdir/jetty-localhost-34085-hadoop-hdfs-3_4_1-tests_jar-_-any-907722254871821266/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T07:39:50,769 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34085} 2024-12-05T07:39:50,769 INFO [Time-limited test {}] server.Server(415): Started @2820ms 2024-12-05T07:39:51,363 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T07:39:51,372 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T07:39:51,373 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T07:39:51,373 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T07:39:51,374 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T07:39:51,375 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/hadoop.log.dir/,AVAILABLE} 2024-12-05T07:39:51,375 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T07:39:51,477 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/java.io.tmpdir/jetty-localhost-43345-hadoop-hdfs-3_4_1-tests_jar-_-any-2551119716834980796/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T07:39:51,478 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:43345} 2024-12-05T07:39:51,478 INFO [Time-limited test {}] server.Server(415): Started @3529ms 2024-12-05T07:39:51,533 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T07:39:52,484 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/cluster_8fe0b83e-62de-a3e8-93ab-b871591f10b7/data/data1/current/BP-332223701-172.17.0.2-1733384389686/current, will proceed with Du for space computation calculation, 2024-12-05T07:39:52,488 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/cluster_8fe0b83e-62de-a3e8-93ab-b871591f10b7/data/data2/current/BP-332223701-172.17.0.2-1733384389686/current, will proceed with Du for space computation calculation, 2024-12-05T07:39:52,525 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T07:39:52,564 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbaa6bab06a8f42c8 with lease ID 0xf20e3b267fbd4941: Processing first storage report for DS-851bc754-0066-4a4b-914a-d67c77d12f88 from datanode DatanodeRegistration(127.0.0.1:34247, datanodeUuid=535bd6b0-421d-4e1c-a321-0e2166523bb2, infoPort=34865, infoSecurePort=0, ipcPort=46051, storageInfo=lv=-57;cid=testClusterID;nsid=1776872170;c=1733384389686) 2024-12-05T07:39:52,566 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbaa6bab06a8f42c8 with lease ID 0xf20e3b267fbd4941: from storage DS-851bc754-0066-4a4b-914a-d67c77d12f88 node DatanodeRegistration(127.0.0.1:34247, datanodeUuid=535bd6b0-421d-4e1c-a321-0e2166523bb2, infoPort=34865, infoSecurePort=0, ipcPort=46051, storageInfo=lv=-57;cid=testClusterID;nsid=1776872170;c=1733384389686), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T07:39:52,567 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbaa6bab06a8f42c8 with lease ID 0xf20e3b267fbd4941: Processing first storage report for DS-6436fc8b-5ebc-4179-b57c-39fce0be642b from datanode DatanodeRegistration(127.0.0.1:34247, datanodeUuid=535bd6b0-421d-4e1c-a321-0e2166523bb2, infoPort=34865, infoSecurePort=0, ipcPort=46051, storageInfo=lv=-57;cid=testClusterID;nsid=1776872170;c=1733384389686) 2024-12-05T07:39:52,567 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbaa6bab06a8f42c8 with lease ID 0xf20e3b267fbd4941: from storage DS-6436fc8b-5ebc-4179-b57c-39fce0be642b node DatanodeRegistration(127.0.0.1:34247, datanodeUuid=535bd6b0-421d-4e1c-a321-0e2166523bb2, infoPort=34865, infoSecurePort=0, ipcPort=46051, storageInfo=lv=-57;cid=testClusterID;nsid=1776872170;c=1733384389686), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T07:39:52,576 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5 2024-12-05T07:39:52,642 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/cluster_8fe0b83e-62de-a3e8-93ab-b871591f10b7/zookeeper_0, clientPort=58664, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/cluster_8fe0b83e-62de-a3e8-93ab-b871591f10b7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/cluster_8fe0b83e-62de-a3e8-93ab-b871591f10b7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T07:39:52,652 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58664 2024-12-05T07:39:52,661 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T07:39:52,665 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T07:39:52,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741825_1001 (size=7) 2024-12-05T07:39:53,298 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15 with version=8 2024-12-05T07:39:53,298 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/hbase-staging 2024-12-05T07:39:53,419 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T07:39:53,639 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fc6cd38557f3:0 server-side Connection retries=45 2024-12-05T07:39:53,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T07:39:53,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T07:39:53,654 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T07:39:53,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T07:39:53,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T07:39:53,762 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T07:39:53,811 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T07:39:53,818 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T07:39:53,821 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T07:39:53,841 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2496 (auto-detected) 2024-12-05T07:39:53,842 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T07:39:53,857 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33643 2024-12-05T07:39:53,876 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33643 connecting to ZooKeeper ensemble=127.0.0.1:58664 2024-12-05T07:39:53,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:336430x0, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T07:39:53,958 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33643-0x101a5b862740000 connected 2024-12-05T07:39:54,279 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T07:39:54,281 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T07:39:54,295 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T07:39:54,300 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15, hbase.cluster.distributed=false 2024-12-05T07:39:54,324 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T07:39:54,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33643 2024-12-05T07:39:54,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33643 2024-12-05T07:39:54,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33643 2024-12-05T07:39:54,333 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33643 2024-12-05T07:39:54,333 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33643 2024-12-05T07:39:54,447 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fc6cd38557f3:0 server-side Connection retries=45 2024-12-05T07:39:54,449 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T07:39:54,449 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T07:39:54,450 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T07:39:54,450 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T07:39:54,450 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T07:39:54,453 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T07:39:54,457 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T07:39:54,458 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36693 2024-12-05T07:39:54,460 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36693 connecting to ZooKeeper ensemble=127.0.0.1:58664 2024-12-05T07:39:54,461 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T07:39:54,465 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T07:39:54,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366930x0, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T07:39:54,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T07:39:54,509 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36693-0x101a5b862740001 connected 2024-12-05T07:39:54,512 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T07:39:54,519 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T07:39:54,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T07:39:54,526 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T07:39:54,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36693 2024-12-05T07:39:54,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36693 2024-12-05T07:39:54,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36693 2024-12-05T07:39:54,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36693 2024-12-05T07:39:54,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36693 2024-12-05T07:39:54,545 DEBUG [M:0;fc6cd38557f3:33643 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fc6cd38557f3:33643 2024-12-05T07:39:54,546 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fc6cd38557f3,33643,1733384393481 2024-12-05T07:39:54,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T07:39:54,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T07:39:54,580 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fc6cd38557f3,33643,1733384393481 2024-12-05T07:39:54,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T07:39:54,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:54,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:54,611 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T07:39:54,612 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fc6cd38557f3,33643,1733384393481 from backup master directory 2024-12-05T07:39:54,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T07:39:54,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fc6cd38557f3,33643,1733384393481 2024-12-05T07:39:54,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T07:39:54,624 WARN [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T07:39:54,625 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fc6cd38557f3,33643,1733384393481 2024-12-05T07:39:54,628 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T07:39:54,631 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T07:39:54,680 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/hbase.id] with ID: ff1984c4-b366-4c5a-a4f7-c150e8d1c04b 2024-12-05T07:39:54,681 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/.tmp/hbase.id 2024-12-05T07:39:54,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741826_1002 (size=42) 2024-12-05T07:39:54,692 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/.tmp/hbase.id]:[hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/hbase.id] 2024-12-05T07:39:54,735 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T07:39:54,739 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T07:39:54,756 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-05T07:39:54,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:54,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:54,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741827_1003 (size=196) 2024-12-05T07:39:54,808 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T07:39:54,810 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T07:39:54,823 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:39:54,827 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T07:39:54,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741828_1004 (size=1189) 2024-12-05T07:39:55,275 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store 2024-12-05T07:39:55,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741829_1005 (size=34) 2024-12-05T07:39:55,703 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T07:39:55,706 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:39:55,707 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T07:39:55,708 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T07:39:55,708 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T07:39:55,709 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T07:39:55,709 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T07:39:55,709 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T07:39:55,710 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733384395707Disabling compacts and flushes for region at 1733384395707Disabling writes for close at 1733384395709 (+2 ms)Writing region close event to WAL at 1733384395709Closed at 1733384395709 2024-12-05T07:39:55,713 WARN [master/fc6cd38557f3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/.initializing 2024-12-05T07:39:55,713 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/WALs/fc6cd38557f3,33643,1733384393481 2024-12-05T07:39:55,722 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T07:39:55,737 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc6cd38557f3%2C33643%2C1733384393481, suffix=, logDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/WALs/fc6cd38557f3,33643,1733384393481, archiveDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/oldWALs, maxLogs=10 2024-12-05T07:39:55,758 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/WALs/fc6cd38557f3,33643,1733384393481/fc6cd38557f3%2C33643%2C1733384393481.1733384395741, exclude list is [], retry=0 2024-12-05T07:39:55,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34247,DS-851bc754-0066-4a4b-914a-d67c77d12f88,DISK] 2024-12-05T07:39:55,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T07:39:55,816 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/WALs/fc6cd38557f3,33643,1733384393481/fc6cd38557f3%2C33643%2C1733384393481.1733384395741 2024-12-05T07:39:55,817 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34865:34865)] 2024-12-05T07:39:55,817 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T07:39:55,818 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:39:55,820 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,821 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T07:39:55,889 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:55,892 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T07:39:55,893 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,897 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T07:39:55,897 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:55,898 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:39:55,898 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,901 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T07:39:55,901 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:55,903 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:39:55,903 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,906 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T07:39:55,906 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:55,907 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:39:55,908 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,912 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,913 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,920 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,921 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,927 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T07:39:55,932 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T07:39:55,936 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T07:39:55,938 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67221610, jitterRate=0.0016800463199615479}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T07:39:55,946 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733384395832Initializing all the Stores at 1733384395835 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733384395835Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384395836 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384395836Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384395836Cleaning up temporary data from old regions at 1733384395921 (+85 ms)Region opened successfully at 1733384395945 (+24 ms) 2024-12-05T07:39:55,948 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T07:39:55,981 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794617bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fc6cd38557f3/172.17.0.2:0 2024-12-05T07:39:56,013 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T07:39:56,022 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T07:39:56,023 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T07:39:56,025 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T07:39:56,026 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T07:39:56,030 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-05T07:39:56,030 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T07:39:56,052 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T07:39:56,060 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T07:39:56,076 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T07:39:56,079 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T07:39:56,081 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T07:39:56,090 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T07:39:56,093 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T07:39:56,098 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T07:39:56,107 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T07:39:56,108 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T07:39:56,118 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T07:39:56,138 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T07:39:56,148 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T07:39:56,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T07:39:56,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T07:39:56,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:56,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:56,164 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fc6cd38557f3,33643,1733384393481, sessionid=0x101a5b862740000, setting cluster-up flag (Was=false) 2024-12-05T07:39:56,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:56,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:56,218 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T07:39:56,221 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fc6cd38557f3,33643,1733384393481 2024-12-05T07:39:56,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:56,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:56,268 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T07:39:56,273 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fc6cd38557f3,33643,1733384393481 2024-12-05T07:39:56,283 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T07:39:56,336 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(746): ClusterId : ff1984c4-b366-4c5a-a4f7-c150e8d1c04b 2024-12-05T07:39:56,339 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T07:39:56,350 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T07:39:56,358 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T07:39:56,364 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T07:39:56,368 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T07:39:56,368 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T07:39:56,370 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fc6cd38557f3,33643,1733384393481 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T07:39:56,378 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T07:39:56,379 DEBUG [RS:0;fc6cd38557f3:36693 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d67b1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fc6cd38557f3/172.17.0.2:0 2024-12-05T07:39:56,380 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fc6cd38557f3:0, corePoolSize=5, maxPoolSize=5 2024-12-05T07:39:56,380 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fc6cd38557f3:0, corePoolSize=5, maxPoolSize=5 2024-12-05T07:39:56,380 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fc6cd38557f3:0, corePoolSize=5, maxPoolSize=5 2024-12-05T07:39:56,380 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fc6cd38557f3:0, corePoolSize=5, maxPoolSize=5 2024-12-05T07:39:56,380 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fc6cd38557f3:0, corePoolSize=10, maxPoolSize=10 2024-12-05T07:39:56,381 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,381 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fc6cd38557f3:0, corePoolSize=2, maxPoolSize=2 2024-12-05T07:39:56,381 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,383 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733384426383 2024-12-05T07:39:56,385 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T07:39:56,386 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T07:39:56,386 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T07:39:56,387 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T07:39:56,391 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T07:39:56,391 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T07:39:56,391 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T07:39:56,392 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T07:39:56,392 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:56,392 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T07:39:56,393 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fc6cd38557f3:36693 2024-12-05T07:39:56,392 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,396 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T07:39:56,397 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T07:39:56,397 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T07:39:56,397 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T07:39:56,397 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T07:39:56,398 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T07:39:56,400 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(2659): reportForDuty to master=fc6cd38557f3,33643,1733384393481 with port=36693, startcode=1733384394401 2024-12-05T07:39:56,402 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T07:39:56,403 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T07:39:56,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741831_1007 (size=1321) 2024-12-05T07:39:56,406 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fc6cd38557f3:0:becomeActiveMaster-HFileCleaner.large.0-1733384396405,5,FailOnTimeoutGroup] 2024-12-05T07:39:56,407 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T07:39:56,407 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15 2024-12-05T07:39:56,408 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fc6cd38557f3:0:becomeActiveMaster-HFileCleaner.small.0-1733384396407,5,FailOnTimeoutGroup] 2024-12-05T07:39:56,408 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,409 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T07:39:56,410 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,411 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,413 DEBUG [RS:0;fc6cd38557f3:36693 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T07:39:56,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741832_1008 (size=32) 2024-12-05T07:39:56,451 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40349, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T07:39:56,457 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33643 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fc6cd38557f3,36693,1733384394401 2024-12-05T07:39:56,460 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33643 {}] master.ServerManager(517): Registering regionserver=fc6cd38557f3,36693,1733384394401 2024-12-05T07:39:56,473 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15 2024-12-05T07:39:56,473 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36945 2024-12-05T07:39:56,473 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T07:39:56,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T07:39:56,502 DEBUG [RS:0;fc6cd38557f3:36693 {}] zookeeper.ZKUtil(111): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fc6cd38557f3,36693,1733384394401 2024-12-05T07:39:56,503 WARN [RS:0;fc6cd38557f3:36693 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T07:39:56,503 INFO [RS:0;fc6cd38557f3:36693 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T07:39:56,503 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/WALs/fc6cd38557f3,36693,1733384394401 2024-12-05T07:39:56,505 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fc6cd38557f3,36693,1733384394401] 2024-12-05T07:39:56,533 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T07:39:56,546 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T07:39:56,550 INFO [RS:0;fc6cd38557f3:36693 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T07:39:56,550 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,551 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T07:39:56,556 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T07:39:56,558 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,558 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,558 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,558 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,559 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,559 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,559 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fc6cd38557f3:0, corePoolSize=2, maxPoolSize=2 2024-12-05T07:39:56,559 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,559 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,559 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,560 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,560 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,560 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fc6cd38557f3:0, corePoolSize=1, maxPoolSize=1 2024-12-05T07:39:56,560 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fc6cd38557f3:0, corePoolSize=3, maxPoolSize=3 2024-12-05T07:39:56,561 DEBUG [RS:0;fc6cd38557f3:36693 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0, corePoolSize=3, maxPoolSize=3 2024-12-05T07:39:56,562 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,562 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,562 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,562 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,563 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,563 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=fc6cd38557f3,36693,1733384394401-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T07:39:56,579 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T07:39:56,580 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=fc6cd38557f3,36693,1733384394401-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,581 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,581 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.Replication(171): fc6cd38557f3,36693,1733384394401 started 2024-12-05T07:39:56,597 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:56,597 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1482): Serving as fc6cd38557f3,36693,1733384394401, RpcServer on fc6cd38557f3/172.17.0.2:36693, sessionid=0x101a5b862740001 2024-12-05T07:39:56,598 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T07:39:56,598 DEBUG [RS:0;fc6cd38557f3:36693 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fc6cd38557f3,36693,1733384394401 2024-12-05T07:39:56,599 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fc6cd38557f3,36693,1733384394401' 2024-12-05T07:39:56,599 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T07:39:56,600 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T07:39:56,600 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T07:39:56,601 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T07:39:56,601 DEBUG [RS:0;fc6cd38557f3:36693 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fc6cd38557f3,36693,1733384394401 2024-12-05T07:39:56,601 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fc6cd38557f3,36693,1733384394401' 2024-12-05T07:39:56,601 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T07:39:56,602 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T07:39:56,602 DEBUG [RS:0;fc6cd38557f3:36693 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T07:39:56,602 INFO [RS:0;fc6cd38557f3:36693 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T07:39:56,602 INFO [RS:0;fc6cd38557f3:36693 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T07:39:56,711 INFO [RS:0;fc6cd38557f3:36693 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T07:39:56,717 INFO [RS:0;fc6cd38557f3:36693 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc6cd38557f3%2C36693%2C1733384394401, suffix=, logDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/WALs/fc6cd38557f3,36693,1733384394401, archiveDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/oldWALs, maxLogs=32 2024-12-05T07:39:56,730 DEBUG [RS:0;fc6cd38557f3:36693 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/WALs/fc6cd38557f3,36693,1733384394401/fc6cd38557f3%2C36693%2C1733384394401.1733384396719, exclude list is [], retry=0 2024-12-05T07:39:56,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34247,DS-851bc754-0066-4a4b-914a-d67c77d12f88,DISK] 2024-12-05T07:39:56,738 INFO [RS:0;fc6cd38557f3:36693 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/WALs/fc6cd38557f3,36693,1733384394401/fc6cd38557f3%2C36693%2C1733384394401.1733384396719 2024-12-05T07:39:56,738 DEBUG [RS:0;fc6cd38557f3:36693 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34865:34865)] 2024-12-05T07:39:56,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:39:56,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T07:39:56,832 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T07:39:56,832 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:56,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T07:39:56,834 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T07:39:56,836 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T07:39:56,836 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:56,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T07:39:56,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T07:39:56,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T07:39:56,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:56,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T07:39:56,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T07:39:56,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T07:39:56,843 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:56,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T07:39:56,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T07:39:56,846 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740 2024-12-05T07:39:56,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740 2024-12-05T07:39:56,849 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T07:39:56,849 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T07:39:56,850 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T07:39:56,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T07:39:56,857 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T07:39:56,858 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62210515, jitterRate=-0.07299108803272247}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T07:39:56,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733384396823Initializing all the Stores at 1733384396826 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733384396826Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733384396827 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384396827Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733384396827Cleaning up temporary data from old regions at 1733384396849 (+22 ms)Region opened successfully at 1733384396859 (+10 ms) 2024-12-05T07:39:56,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T07:39:56,860 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T07:39:56,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T07:39:56,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T07:39:56,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T07:39:56,861 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T07:39:56,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733384396859Disabling compacts and flushes for region at 1733384396859Disabling writes for close at 1733384396860 (+1 ms)Writing region close event to WAL at 1733384396861 (+1 ms)Closed at 1733384396861 2024-12-05T07:39:56,864 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T07:39:56,865 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T07:39:56,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T07:39:56,877 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T07:39:56,880 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T07:39:57,033 DEBUG [fc6cd38557f3:33643 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T07:39:57,046 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:39:57,054 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc6cd38557f3,36693,1733384394401, state=OPENING 2024-12-05T07:39:57,107 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T07:39:57,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:57,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:39:57,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T07:39:57,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T07:39:57,124 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T07:39:57,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:39:57,299 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T07:39:57,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43171, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T07:39:57,313 INFO [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T07:39:57,313 INFO [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T07:39:57,314 INFO [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T07:39:57,317 INFO [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fc6cd38557f3%2C36693%2C1733384394401.meta, suffix=.meta, logDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/WALs/fc6cd38557f3,36693,1733384394401, archiveDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/oldWALs, maxLogs=32 2024-12-05T07:39:57,330 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/WALs/fc6cd38557f3,36693,1733384394401/fc6cd38557f3%2C36693%2C1733384394401.meta.1733384397318.meta, exclude list is [], retry=0 2024-12-05T07:39:57,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34247,DS-851bc754-0066-4a4b-914a-d67c77d12f88,DISK] 2024-12-05T07:39:57,336 INFO [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/WALs/fc6cd38557f3,36693,1733384394401/fc6cd38557f3%2C36693%2C1733384394401.meta.1733384397318.meta 2024-12-05T07:39:57,337 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34865:34865)] 2024-12-05T07:39:57,337 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T07:39:57,339 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T07:39:57,341 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T07:39:57,345 INFO [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T07:39:57,349 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T07:39:57,349 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:39:57,350 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T07:39:57,350 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T07:39:57,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T07:39:57,354 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T07:39:57,354 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:57,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T07:39:57,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T07:39:57,357 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T07:39:57,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:57,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T07:39:57,358 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T07:39:57,359 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T07:39:57,359 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:57,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T07:39:57,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T07:39:57,362 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T07:39:57,362 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:57,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T07:39:57,363 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T07:39:57,364 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740 2024-12-05T07:39:57,366 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740 2024-12-05T07:39:57,369 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T07:39:57,369 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T07:39:57,370 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T07:39:57,372 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T07:39:57,374 INFO [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70514239, jitterRate=0.05074404180049896}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T07:39:57,374 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T07:39:57,375 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733384397350Writing region info on filesystem at 1733384397350Initializing all the Stores at 1733384397352 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733384397352Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733384397353 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384397353Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733384397353Cleaning up temporary data from old regions at 1733384397369 (+16 ms)Running coprocessor post-open hooks at 1733384397374 (+5 ms)Region opened successfully at 1733384397375 (+1 ms) 2024-12-05T07:39:57,381 INFO [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733384397292 2024-12-05T07:39:57,390 DEBUG [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T07:39:57,391 INFO [RS_OPEN_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T07:39:57,392 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:39:57,394 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fc6cd38557f3,36693,1733384394401, state=OPEN 2024-12-05T07:39:57,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T07:39:57,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T07:39:57,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T07:39:57,443 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T07:39:57,443 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:39:57,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T07:39:57,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fc6cd38557f3,36693,1733384394401 in 316 msec 2024-12-05T07:39:57,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T07:39:57,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 584 msec 2024-12-05T07:39:57,463 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T07:39:57,463 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T07:39:57,483 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:39:57,484 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:39:57,511 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:39:57,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48913, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:39:57,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2280 sec 2024-12-05T07:39:57,540 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733384397540, completionTime=-1 2024-12-05T07:39:57,544 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T07:39:57,544 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T07:39:57,572 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-05T07:39:57,572 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733384457572 2024-12-05T07:39:57,572 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733384517572 2024-12-05T07:39:57,572 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-12-05T07:39:57,575 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc6cd38557f3,33643,1733384393481-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:57,575 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc6cd38557f3,33643,1733384393481-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:57,575 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc6cd38557f3,33643,1733384393481-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:57,577 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fc6cd38557f3:33643, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:57,578 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:57,578 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:57,584 DEBUG [master/fc6cd38557f3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T07:39:57,631 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.006sec 2024-12-05T07:39:57,633 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T07:39:57,635 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T07:39:57,636 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T07:39:57,637 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T07:39:57,637 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T07:39:57,638 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc6cd38557f3,33643,1733384393481-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T07:39:57,639 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc6cd38557f3,33643,1733384393481-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T07:39:57,658 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T07:39:57,659 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T07:39:57,661 INFO [master/fc6cd38557f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fc6cd38557f3,33643,1733384393481-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T07:39:57,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:39:57,664 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T07:39:57,664 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T07:39:57,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:39:57,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:39:57,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:39:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:39:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:39:57,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5670729d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:39:57,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:39:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:39:57,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:39:57,688 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:39:57,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:39:57,692 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:39:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:39:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:39:57,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:39:57,723 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fc6cd38557f3,33643,1733384393481 2024-12-05T07:39:57,734 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=345, ProcessCount=11, AvailableMemoryMB=7679 2024-12-05T07:39:57,742 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T07:39:57,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fc6cd38557f3,33643,1733384393481 2024-12-05T07:39:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@385eccdd 2024-12-05T07:39:57,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T07:39:57,757 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T07:39:57,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T07:39:57,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T07:39:57,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T07:39:57,779 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T07:39:57,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-05T07:39:57,782 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:57,786 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T07:39:57,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T07:39:57,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741835_1011 (size=959) 2024-12-05T07:39:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T07:39:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T07:39:58,202 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15 2024-12-05T07:39:58,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741836_1012 (size=53) 2024-12-05T07:39:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T07:39:58,627 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:39:58,627 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 27ae0e4955183fa53f189012e681dac5, disabling compactions & flushes 2024-12-05T07:39:58,627 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:39:58,627 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:39:58,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. after waiting 0 ms 2024-12-05T07:39:58,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:39:58,628 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:39:58,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 27ae0e4955183fa53f189012e681dac5: Waiting for close lock at 1733384398627Disabling compacts and flushes for region at 1733384398627Disabling writes for close at 1733384398628 (+1 ms)Writing region close event to WAL at 1733384398628Closed at 1733384398628 2024-12-05T07:39:58,632 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T07:39:58,637 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733384398632"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733384398632"}]},"ts":"1733384398632"} 2024-12-05T07:39:58,642 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T07:39:58,644 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T07:39:58,646 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384398644"}]},"ts":"1733384398644"} 2024-12-05T07:39:58,649 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T07:39:58,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=27ae0e4955183fa53f189012e681dac5, ASSIGN}] 2024-12-05T07:39:58,654 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=27ae0e4955183fa53f189012e681dac5, ASSIGN 2024-12-05T07:39:58,655 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=27ae0e4955183fa53f189012e681dac5, ASSIGN; state=OFFLINE, location=fc6cd38557f3,36693,1733384394401; forceNewPlan=false, retain=false 2024-12-05T07:39:58,808 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=27ae0e4955183fa53f189012e681dac5, regionState=OPENING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:39:58,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=27ae0e4955183fa53f189012e681dac5, ASSIGN because future has completed 2024-12-05T07:39:58,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:39:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T07:39:58,979 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:39:58,980 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} 2024-12-05T07:39:58,980 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:58,980 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:39:58,981 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:58,981 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:58,984 INFO [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:58,986 INFO [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27ae0e4955183fa53f189012e681dac5 columnFamilyName A 2024-12-05T07:39:58,986 DEBUG [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:58,987 INFO [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] regionserver.HStore(327): Store=27ae0e4955183fa53f189012e681dac5/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:39:58,987 INFO [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:58,989 INFO [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27ae0e4955183fa53f189012e681dac5 columnFamilyName B 2024-12-05T07:39:58,989 DEBUG [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:58,990 INFO [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] regionserver.HStore(327): Store=27ae0e4955183fa53f189012e681dac5/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:39:58,990 INFO [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:58,992 INFO [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 27ae0e4955183fa53f189012e681dac5 columnFamilyName C 2024-12-05T07:39:58,993 DEBUG [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:39:58,993 INFO [StoreOpener-27ae0e4955183fa53f189012e681dac5-1 {}] regionserver.HStore(327): Store=27ae0e4955183fa53f189012e681dac5/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:39:58,994 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:58,995 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:58,996 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:58,998 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:58,998 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:58,999 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T07:39:59,001 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:59,004 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T07:39:59,005 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 27ae0e4955183fa53f189012e681dac5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70535714, jitterRate=0.05106404423713684}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T07:39:59,005 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:39:59,006 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 27ae0e4955183fa53f189012e681dac5: Running coprocessor pre-open hook at 1733384398981Writing region info on filesystem at 1733384398981Initializing all the Stores at 1733384398983 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384398983Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384398983Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384398983Cleaning up temporary data from old regions at 1733384398998 (+15 ms)Running coprocessor post-open hooks at 1733384399005 (+7 ms)Region opened successfully at 1733384399006 (+1 ms) 2024-12-05T07:39:59,008 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., pid=6, masterSystemTime=1733384398972 2024-12-05T07:39:59,011 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:39:59,011 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:39:59,012 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=27ae0e4955183fa53f189012e681dac5, regionState=OPEN, openSeqNum=2, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:39:59,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:39:59,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T07:39:59,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 in 200 msec 2024-12-05T07:39:59,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T07:39:59,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=27ae0e4955183fa53f189012e681dac5, ASSIGN in 371 msec 2024-12-05T07:39:59,027 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T07:39:59,027 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384399027"}]},"ts":"1733384399027"} 2024-12-05T07:39:59,030 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T07:39:59,031 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T07:39:59,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2630 sec 2024-12-05T07:39:59,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T07:39:59,947 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:39:59,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:39:59,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:39:59,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:39:59,954 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:39:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:39:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:39:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70cdc6cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:39:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:39:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:39:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:39:59,958 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:39:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1806c24b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:39:59,961 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:39:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:39:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:39:59,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:39:59,972 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T07:39:59,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fc6cd38557f3,33643,1733384393481 2024-12-05T07:39:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78210a9b 2024-12-05T07:39:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T07:39:59,980 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T07:39:59,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:39:59,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:39:59,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:39:59,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:39:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:39:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:39:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b084ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:39:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:39:59,990 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:39:59,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:39:59,996 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:39:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:39:59,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:39:59,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:00,000 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:00,002 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45101745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:00,006 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c55d99d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:00,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:00,038 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:00,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:00,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:00,044 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39deefb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:00,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:00,053 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ccbe83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:00,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:00,062 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:00,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5970e36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:00,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:00,067 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:00,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644e9151, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:00,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:00,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:00,071 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:00,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c82a8df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:00,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:00,081 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:00,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641e220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:00,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:00,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:00,085 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f3e22b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3111333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:00,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:00,093 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20584f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:00,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:00,097 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7aa9af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:00,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:00,103 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c287047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:00,106 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c9a155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,111 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:00,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:00,112 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:00,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1608b68b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:00,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:00,116 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5b0127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:00,123 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:00,123 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:00,123 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:00,124 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:00,127 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:00,128 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:00,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:00,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:00,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:00,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:00,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:00,140 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:00,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:00,146 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-05T07:40:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:00,149 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:00,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:00,152 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T07:40:00,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:00,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:00,154 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:00,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:00,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:00,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:00,163 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:00,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:00,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:00,175 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:00,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:00,192 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:00,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:00,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T07:40:00,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T07:40:00,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T07:40:00,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:00,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:00,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:00,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/1185a997cd6d400da34d8fe64deddaf3 is 50, key is test_row_0/A:col10/1733384400234/Put/seqid=0 2024-12-05T07:40:00,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741837_1013 (size=12001) 2024-12-05T07:40:00,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/1185a997cd6d400da34d8fe64deddaf3 2024-12-05T07:40:00,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384460436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384460446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384460449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T07:40:00,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384460452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384460452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T07:40:00,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/f00bb68559e34bd3a8e839c34b31b9ac is 50, key is test_row_0/B:col10/1733384400234/Put/seqid=0 2024-12-05T07:40:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:00,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:00,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741838_1014 (size=12001) 2024-12-05T07:40:00,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384460607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384460608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384460609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384460609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384460609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T07:40:00,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:00,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:00,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:00,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T07:40:00,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384460826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384460827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384460827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384460828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384460828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:00,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T07:40:00,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:00,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:00,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:00,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/f00bb68559e34bd3a8e839c34b31b9ac 2024-12-05T07:40:00,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T07:40:00,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:00,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:00,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:01,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/c0f39cfda9b54613a2096328363f9251 is 50, key is test_row_0/C:col10/1733384400234/Put/seqid=0 2024-12-05T07:40:01,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741839_1015 (size=12001) 2024-12-05T07:40:01,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384461138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384461138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384461138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384461139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384461139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T07:40:01,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:01,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:01,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:01,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:01,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T07:40:01,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T07:40:01,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:01,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:01,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:01,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:01,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:01,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/c0f39cfda9b54613a2096328363f9251 2024-12-05T07:40:01,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T07:40:01,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:01,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:01,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:01,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:01,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:01,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/1185a997cd6d400da34d8fe64deddaf3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/1185a997cd6d400da34d8fe64deddaf3 2024-12-05T07:40:01,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/1185a997cd6d400da34d8fe64deddaf3, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T07:40:01,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/f00bb68559e34bd3a8e839c34b31b9ac as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/f00bb68559e34bd3a8e839c34b31b9ac 2024-12-05T07:40:01,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/f00bb68559e34bd3a8e839c34b31b9ac, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T07:40:01,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/c0f39cfda9b54613a2096328363f9251 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c0f39cfda9b54613a2096328363f9251 2024-12-05T07:40:01,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c0f39cfda9b54613a2096328363f9251, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T07:40:01,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 27ae0e4955183fa53f189012e681dac5 in 1316ms, sequenceid=14, compaction requested=false 2024-12-05T07:40:01,574 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T07:40:01,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:01,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T07:40:01,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:01,620 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T07:40:01,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/d08c84990b1a4e57970ec51b355707db is 50, key is test_row_0/A:col10/1733384400445/Put/seqid=0 2024-12-05T07:40:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:01,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:01,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741840_1016 (size=12001) 2024-12-05T07:40:01,707 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/d08c84990b1a4e57970ec51b355707db 2024-12-05T07:40:01,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/9af2c5363c3c4aef9223ebcef11dc8ac is 50, key is test_row_0/B:col10/1733384400445/Put/seqid=0 2024-12-05T07:40:01,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384461725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384461728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384461743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384461743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384461747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741841_1017 (size=12001) 2024-12-05T07:40:01,792 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/9af2c5363c3c4aef9223ebcef11dc8ac 2024-12-05T07:40:01,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/0db9fcc7c63544cdbebf5d8bf9786787 is 50, key is test_row_0/C:col10/1733384400445/Put/seqid=0 2024-12-05T07:40:01,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384461856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384461857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741842_1018 (size=12001) 2024-12-05T07:40:01,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384461867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384461867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:01,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384461877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384462077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384462077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,085 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384462087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384462087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384462097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,274 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/0db9fcc7c63544cdbebf5d8bf9786787 2024-12-05T07:40:02,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/d08c84990b1a4e57970ec51b355707db as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d08c84990b1a4e57970ec51b355707db 2024-12-05T07:40:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T07:40:02,304 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d08c84990b1a4e57970ec51b355707db, entries=150, sequenceid=38, filesize=11.7 K 2024-12-05T07:40:02,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/9af2c5363c3c4aef9223ebcef11dc8ac as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9af2c5363c3c4aef9223ebcef11dc8ac 2024-12-05T07:40:02,320 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9af2c5363c3c4aef9223ebcef11dc8ac, entries=150, sequenceid=38, filesize=11.7 K 2024-12-05T07:40:02,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/0db9fcc7c63544cdbebf5d8bf9786787 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/0db9fcc7c63544cdbebf5d8bf9786787 2024-12-05T07:40:02,339 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/0db9fcc7c63544cdbebf5d8bf9786787, entries=150, sequenceid=38, filesize=11.7 K 2024-12-05T07:40:02,343 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 27ae0e4955183fa53f189012e681dac5 in 723ms, sequenceid=38, compaction requested=false 2024-12-05T07:40:02,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:02,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:02,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-05T07:40:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-05T07:40:02,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T07:40:02,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1940 sec 2024-12-05T07:40:02,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.2130 sec 2024-12-05T07:40:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:02,403 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:02,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T07:40:02,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/9c3d497eefa241ffbdbc9433883c59e6 is 50, key is test_row_0/A:col10/1733384402399/Put/seqid=0 2024-12-05T07:40:02,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741843_1019 (size=16681) 2024-12-05T07:40:02,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/9c3d497eefa241ffbdbc9433883c59e6 2024-12-05T07:40:02,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384462454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384462458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384462469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384462469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384462470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/9538e65d9bd94aa6a9723982a6b79b71 is 50, key is test_row_0/B:col10/1733384402399/Put/seqid=0 2024-12-05T07:40:02,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741844_1020 (size=12001) 2024-12-05T07:40:02,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384462577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384462577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384462577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384462587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384462588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384462798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384462798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384462798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384462803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384462807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:02,860 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T07:40:02,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/9538e65d9bd94aa6a9723982a6b79b71 2024-12-05T07:40:02,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/5c00ecc705504deb822d260a17898e11 is 50, key is test_row_0/C:col10/1733384402399/Put/seqid=0 2024-12-05T07:40:02,976 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T07:40:03,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741845_1021 (size=12001) 2024-12-05T07:40:03,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/5c00ecc705504deb822d260a17898e11 2024-12-05T07:40:03,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/9c3d497eefa241ffbdbc9433883c59e6 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/9c3d497eefa241ffbdbc9433883c59e6 2024-12-05T07:40:03,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/9c3d497eefa241ffbdbc9433883c59e6, entries=250, sequenceid=53, filesize=16.3 K 2024-12-05T07:40:03,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/9538e65d9bd94aa6a9723982a6b79b71 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9538e65d9bd94aa6a9723982a6b79b71 2024-12-05T07:40:03,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9538e65d9bd94aa6a9723982a6b79b71, entries=150, sequenceid=53, filesize=11.7 K 2024-12-05T07:40:03,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/5c00ecc705504deb822d260a17898e11 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/5c00ecc705504deb822d260a17898e11 2024-12-05T07:40:03,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/5c00ecc705504deb822d260a17898e11, entries=150, sequenceid=53, filesize=11.7 K 2024-12-05T07:40:03,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 27ae0e4955183fa53f189012e681dac5 in 701ms, sequenceid=53, compaction requested=true 2024-12-05T07:40:03,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:03,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:03,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:03,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:03,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:03,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:03,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:03,127 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:03,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-05T07:40:03,129 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:03,130 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:03,133 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:03,135 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/B is initiating minor compaction (all files) 2024-12-05T07:40:03,135 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/B in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:03,137 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/f00bb68559e34bd3a8e839c34b31b9ac, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9af2c5363c3c4aef9223ebcef11dc8ac, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9538e65d9bd94aa6a9723982a6b79b71] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=35.2 K 2024-12-05T07:40:03,137 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:03,137 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/A is initiating minor compaction (all files) 2024-12-05T07:40:03,137 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/A in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:03,138 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/1185a997cd6d400da34d8fe64deddaf3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d08c84990b1a4e57970ec51b355707db, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/9c3d497eefa241ffbdbc9433883c59e6] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=39.7 K 2024-12-05T07:40:03,139 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f00bb68559e34bd3a8e839c34b31b9ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733384400232 2024-12-05T07:40:03,139 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1185a997cd6d400da34d8fe64deddaf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733384400232 2024-12-05T07:40:03,141 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d08c84990b1a4e57970ec51b355707db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733384400429 2024-12-05T07:40:03,141 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af2c5363c3c4aef9223ebcef11dc8ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733384400429 2024-12-05T07:40:03,142 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c3d497eefa241ffbdbc9433883c59e6, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733384401688 2024-12-05T07:40:03,142 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9538e65d9bd94aa6a9723982a6b79b71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733384401688 2024-12-05T07:40:03,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/d8a33272a2ac4365ac016ebdf6a56696 is 50, key is test_row_0/A:col10/1733384402464/Put/seqid=0 2024-12-05T07:40:03,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384463178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384463181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384463191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384463199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741846_1022 (size=12001) 2024-12-05T07:40:03,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/d8a33272a2ac4365ac016ebdf6a56696 2024-12-05T07:40:03,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384463201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,242 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#B#compaction#10 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:03,243 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/079b3d1276c04336a0f773c12e8644bb is 50, key is test_row_0/B:col10/1733384402399/Put/seqid=0 2024-12-05T07:40:03,245 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#A#compaction#11 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:03,246 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/68413359cebd4798bf0a91eb236d3d92 is 50, key is test_row_0/A:col10/1733384402399/Put/seqid=0 2024-12-05T07:40:03,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/4e39f7b2c862460cbcc16996ddcc256c is 50, key is test_row_0/B:col10/1733384402464/Put/seqid=0 2024-12-05T07:40:03,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384463306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384463306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741847_1023 (size=12104) 2024-12-05T07:40:03,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384463337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384463337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741848_1024 (size=12104) 2024-12-05T07:40:03,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384463346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741849_1025 (size=12001) 2024-12-05T07:40:03,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/4e39f7b2c862460cbcc16996ddcc256c 2024-12-05T07:40:03,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/43716f4175eb4306a5e14ad20ce60a46 is 50, key is test_row_0/C:col10/1733384402464/Put/seqid=0 2024-12-05T07:40:03,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741850_1026 (size=12001) 2024-12-05T07:40:03,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/43716f4175eb4306a5e14ad20ce60a46 2024-12-05T07:40:03,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/d8a33272a2ac4365ac016ebdf6a56696 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d8a33272a2ac4365ac016ebdf6a56696 2024-12-05T07:40:03,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d8a33272a2ac4365ac016ebdf6a56696, entries=150, sequenceid=76, filesize=11.7 K 2024-12-05T07:40:03,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/4e39f7b2c862460cbcc16996ddcc256c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/4e39f7b2c862460cbcc16996ddcc256c 2024-12-05T07:40:03,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/4e39f7b2c862460cbcc16996ddcc256c, entries=150, sequenceid=76, filesize=11.7 K 2024-12-05T07:40:03,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/43716f4175eb4306a5e14ad20ce60a46 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/43716f4175eb4306a5e14ad20ce60a46 2024-12-05T07:40:03,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/43716f4175eb4306a5e14ad20ce60a46, entries=150, sequenceid=76, filesize=11.7 K 2024-12-05T07:40:03,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 27ae0e4955183fa53f189012e681dac5 in 414ms, sequenceid=76, compaction requested=true 2024-12-05T07:40:03,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:03,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:A, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:03,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T07:40:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:B, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-05T07:40:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-05T07:40:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:03,555 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:03,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-05T07:40:03,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/118a2f5d1c0349a3895f221f9c65672f is 50, key is test_row_0/A:col10/1733384403550/Put/seqid=0 2024-12-05T07:40:03,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741851_1027 (size=16681) 2024-12-05T07:40:03,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384463634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384463635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384463638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384463643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384463652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,755 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/68413359cebd4798bf0a91eb236d3d92 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/68413359cebd4798bf0a91eb236d3d92 2024-12-05T07:40:03,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384463766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384463766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384463767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,780 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/079b3d1276c04336a0f773c12e8644bb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/079b3d1276c04336a0f773c12e8644bb 2024-12-05T07:40:03,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384463767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384463776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,797 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/B of 27ae0e4955183fa53f189012e681dac5 into 079b3d1276c04336a0f773c12e8644bb(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:03,797 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/A of 27ae0e4955183fa53f189012e681dac5 into 68413359cebd4798bf0a91eb236d3d92(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:03,797 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:03,797 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:03,800 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/A, priority=13, startTime=1733384403106; duration=0sec 2024-12-05T07:40:03,800 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/B, priority=13, startTime=1733384403126; duration=0sec 2024-12-05T07:40:03,801 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-05T07:40:03,801 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:03,801 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:03,801 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:03,801 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-05T07:40:03,802 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:03,802 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:03,804 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:03,804 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/C is initiating minor compaction (all files) 2024-12-05T07:40:03,804 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/C in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:03,804 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c0f39cfda9b54613a2096328363f9251, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/0db9fcc7c63544cdbebf5d8bf9786787, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/5c00ecc705504deb822d260a17898e11, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/43716f4175eb4306a5e14ad20ce60a46] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=46.9 K 2024-12-05T07:40:03,805 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-05T07:40:03,805 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T07:40:03,805 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T07:40:03,805 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. because compaction request was cancelled 2024-12-05T07:40:03,805 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:C 2024-12-05T07:40:03,805 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting c0f39cfda9b54613a2096328363f9251, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733384400232 2024-12-05T07:40:03,805 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-05T07:40:03,806 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db9fcc7c63544cdbebf5d8bf9786787, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1733384400429 2024-12-05T07:40:03,807 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T07:40:03,807 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c00ecc705504deb822d260a17898e11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733384401688 2024-12-05T07:40:03,807 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T07:40:03,807 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. because compaction request was cancelled 2024-12-05T07:40:03,807 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:03,807 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-05T07:40:03,808 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 43716f4175eb4306a5e14ad20ce60a46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733384402464 2024-12-05T07:40:03,809 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T07:40:03,809 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T07:40:03,811 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T07:40:03,811 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T07:40:03,811 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. because compaction request was cancelled 2024-12-05T07:40:03,811 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:03,811 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T07:40:03,811 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T07:40:03,812 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T07:40:03,812 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T07:40:03,813 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T07:40:03,813 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T07:40:03,845 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#C#compaction#15 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:03,846 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/58ce132f8d92435e8f039830cd0bfd4e is 50, key is test_row_0/C:col10/1733384402464/Put/seqid=0 2024-12-05T07:40:03,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741852_1028 (size=12139) 2024-12-05T07:40:03,883 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/58ce132f8d92435e8f039830cd0bfd4e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/58ce132f8d92435e8f039830cd0bfd4e 2024-12-05T07:40:03,897 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/C of 27ae0e4955183fa53f189012e681dac5 into 58ce132f8d92435e8f039830cd0bfd4e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:03,897 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:03,897 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/C, priority=12, startTime=1733384403546; duration=0sec 2024-12-05T07:40:03,897 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:03,897 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:C 2024-12-05T07:40:03,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384463986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384463986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384463986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384463987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384463987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:03,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:04,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/118a2f5d1c0349a3895f221f9c65672f 2024-12-05T07:40:04,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/6a54f43c3ae1432088b5fe1f408d2afe is 50, key is test_row_0/B:col10/1733384403550/Put/seqid=0 2024-12-05T07:40:04,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741853_1029 (size=12001) 2024-12-05T07:40:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384464297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384464297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T07:40:04,308 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:04,309 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:04,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:04,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384464296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:04,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384464307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:04,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:04,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384464307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:04,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:04,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-05T07:40:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T07:40:04,323 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:04,325 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:04,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T07:40:04,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T07:40:04,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:04,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:04,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:04,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:04,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:04,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/6a54f43c3ae1432088b5fe1f408d2afe 2024-12-05T07:40:04,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/52777ccf3c1847d6a85a12c184b27d2c is 50, key is test_row_0/C:col10/1733384403550/Put/seqid=0 2024-12-05T07:40:04,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741854_1030 (size=12001) 2024-12-05T07:40:04,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/52777ccf3c1847d6a85a12c184b27d2c 2024-12-05T07:40:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T07:40:04,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T07:40:04,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:04,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:04,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:04,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:04,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/118a2f5d1c0349a3895f221f9c65672f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/118a2f5d1c0349a3895f221f9c65672f 2024-12-05T07:40:04,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/118a2f5d1c0349a3895f221f9c65672f, entries=250, sequenceid=91, filesize=16.3 K 2024-12-05T07:40:04,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/6a54f43c3ae1432088b5fe1f408d2afe as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/6a54f43c3ae1432088b5fe1f408d2afe 2024-12-05T07:40:04,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/6a54f43c3ae1432088b5fe1f408d2afe, entries=150, sequenceid=91, filesize=11.7 K 2024-12-05T07:40:04,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/52777ccf3c1847d6a85a12c184b27d2c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/52777ccf3c1847d6a85a12c184b27d2c 2024-12-05T07:40:04,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/52777ccf3c1847d6a85a12c184b27d2c, entries=150, sequenceid=91, filesize=11.7 K 2024-12-05T07:40:04,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 27ae0e4955183fa53f189012e681dac5 in 1180ms, sequenceid=91, compaction requested=true 2024-12-05T07:40:04,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:04,736 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:04,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:04,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:04,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:04,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:04,736 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:04,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:04,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:04,738 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:04,738 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:04,738 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/B is initiating minor compaction (all files) 2024-12-05T07:40:04,738 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/A is initiating minor compaction (all files) 2024-12-05T07:40:04,738 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/A in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:04,738 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/B in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:04,738 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/079b3d1276c04336a0f773c12e8644bb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/4e39f7b2c862460cbcc16996ddcc256c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/6a54f43c3ae1432088b5fe1f408d2afe] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=35.3 K 2024-12-05T07:40:04,738 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/68413359cebd4798bf0a91eb236d3d92, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d8a33272a2ac4365ac016ebdf6a56696, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/118a2f5d1c0349a3895f221f9c65672f] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=39.8 K 2024-12-05T07:40:04,739 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 079b3d1276c04336a0f773c12e8644bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733384401688 2024-12-05T07:40:04,739 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68413359cebd4798bf0a91eb236d3d92, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733384401688 2024-12-05T07:40:04,741 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8a33272a2ac4365ac016ebdf6a56696, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733384402464 2024-12-05T07:40:04,741 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e39f7b2c862460cbcc16996ddcc256c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733384402464 2024-12-05T07:40:04,742 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a54f43c3ae1432088b5fe1f408d2afe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733384403175 2024-12-05T07:40:04,742 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 118a2f5d1c0349a3895f221f9c65672f, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733384403175 2024-12-05T07:40:04,765 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:04,766 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/1f22c6eec3844e91aad875cfcf92e2fb is 50, key is test_row_0/B:col10/1733384403550/Put/seqid=0 2024-12-05T07:40:04,776 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#A#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:04,776 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/240c30feed4a40fb994baad76191adba is 50, key is test_row_0/A:col10/1733384403550/Put/seqid=0 2024-12-05T07:40:04,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T07:40:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:04,797 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T07:40:04,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741855_1031 (size=12207) 2024-12-05T07:40:04,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/a82e7fe6410942888a6782564442d564 is 50, key is test_row_0/A:col10/1733384403648/Put/seqid=0 2024-12-05T07:40:04,831 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/1f22c6eec3844e91aad875cfcf92e2fb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/1f22c6eec3844e91aad875cfcf92e2fb 2024-12-05T07:40:04,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:04,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:04,844 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/B of 27ae0e4955183fa53f189012e681dac5 into 1f22c6eec3844e91aad875cfcf92e2fb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:04,845 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:04,845 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/B, priority=13, startTime=1733384404736; duration=0sec 2024-12-05T07:40:04,846 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:04,846 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:04,846 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-05T07:40:04,848 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-05T07:40:04,848 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-05T07:40:04,848 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. because compaction request was cancelled 2024-12-05T07:40:04,848 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:C 2024-12-05T07:40:04,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741856_1032 (size=12207) 2024-12-05T07:40:04,875 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/240c30feed4a40fb994baad76191adba as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/240c30feed4a40fb994baad76191adba 2024-12-05T07:40:04,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:04,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384464865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:04,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741857_1033 (size=12001) 2024-12-05T07:40:04,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:04,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384464867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:04,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384464871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:04,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384464886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:04,895 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/A of 27ae0e4955183fa53f189012e681dac5 into 240c30feed4a40fb994baad76191adba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,896 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:04,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:04,896 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/A, priority=13, startTime=1733384404735; duration=0sec 2024-12-05T07:40:04,896 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:04,896 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:04,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384464890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T07:40:04,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384464997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384465006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384465006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384465006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384464996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384465206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384465226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384465227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384465227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384465227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,303 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/a82e7fe6410942888a6782564442d564 2024-12-05T07:40:05,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/753072476fa4424e881a5fd8c5393594 is 50, key is test_row_0/B:col10/1733384403648/Put/seqid=0 2024-12-05T07:40:05,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741858_1034 (size=12001) 2024-12-05T07:40:05,359 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/753072476fa4424e881a5fd8c5393594 2024-12-05T07:40:05,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/ce65850ebcef4f25b248c935040db891 is 50, key is test_row_0/C:col10/1733384403648/Put/seqid=0 2024-12-05T07:40:05,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741859_1035 (size=12001) 2024-12-05T07:40:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T07:40:05,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384465515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384465536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384465537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384465537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:05,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384465537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:05,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:05,796 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/ce65850ebcef4f25b248c935040db891 2024-12-05T07:40:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/a82e7fe6410942888a6782564442d564 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a82e7fe6410942888a6782564442d564 2024-12-05T07:40:05,826 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a82e7fe6410942888a6782564442d564, entries=150, sequenceid=115, filesize=11.7 K 2024-12-05T07:40:05,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/753072476fa4424e881a5fd8c5393594 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/753072476fa4424e881a5fd8c5393594 2024-12-05T07:40:05,843 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/753072476fa4424e881a5fd8c5393594, entries=150, sequenceid=115, filesize=11.7 K 2024-12-05T07:40:05,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/ce65850ebcef4f25b248c935040db891 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ce65850ebcef4f25b248c935040db891 2024-12-05T07:40:05,891 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ce65850ebcef4f25b248c935040db891, entries=150, sequenceid=115, filesize=11.7 K 2024-12-05T07:40:05,893 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 27ae0e4955183fa53f189012e681dac5 in 1095ms, sequenceid=115, compaction requested=true 2024-12-05T07:40:05,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:05,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:05,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-05T07:40:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-05T07:40:05,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T07:40:05,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5730 sec 2024-12-05T07:40:05,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.5970 sec 2024-12-05T07:40:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:06,030 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:06,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-05T07:40:06,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/e00c8623f92c4b8c89da982592607296 is 50, key is test_row_0/A:col10/1733384404859/Put/seqid=0 2024-12-05T07:40:06,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741860_1036 (size=14491) 2024-12-05T07:40:06,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/e00c8623f92c4b8c89da982592607296 2024-12-05T07:40:06,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384466073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384466078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/64cbef27535643b585b0b3f8b02c6573 is 50, key is test_row_0/B:col10/1733384404859/Put/seqid=0 2024-12-05T07:40:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384466079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384466081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384466079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741861_1037 (size=12101) 2024-12-05T07:40:06,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/64cbef27535643b585b0b3f8b02c6573 2024-12-05T07:40:06,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/80453c9a701c4def9b1f69833bb708b8 is 50, key is test_row_0/C:col10/1733384404859/Put/seqid=0 2024-12-05T07:40:06,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741862_1038 (size=12101) 2024-12-05T07:40:06,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/80453c9a701c4def9b1f69833bb708b8 2024-12-05T07:40:06,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/e00c8623f92c4b8c89da982592607296 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/e00c8623f92c4b8c89da982592607296 2024-12-05T07:40:06,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/e00c8623f92c4b8c89da982592607296, entries=200, sequenceid=134, filesize=14.2 K 2024-12-05T07:40:06,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/64cbef27535643b585b0b3f8b02c6573 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/64cbef27535643b585b0b3f8b02c6573 2024-12-05T07:40:06,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/64cbef27535643b585b0b3f8b02c6573, entries=150, sequenceid=134, filesize=11.8 K 2024-12-05T07:40:06,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/80453c9a701c4def9b1f69833bb708b8 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/80453c9a701c4def9b1f69833bb708b8 2024-12-05T07:40:06,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384466187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384466196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384466197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/80453c9a701c4def9b1f69833bb708b8, entries=150, sequenceid=134, filesize=11.8 K 2024-12-05T07:40:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384466197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 27ae0e4955183fa53f189012e681dac5 in 170ms, sequenceid=134, compaction requested=true 2024-12-05T07:40:06,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:06,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:06,200 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:06,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:06,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:06,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:06,201 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:06,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:06,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:06,202 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:06,202 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:06,202 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/A is initiating minor compaction (all files) 2024-12-05T07:40:06,202 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/B is initiating minor compaction (all files) 2024-12-05T07:40:06,203 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/A in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:06,203 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/B in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:06,203 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/1f22c6eec3844e91aad875cfcf92e2fb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/753072476fa4424e881a5fd8c5393594, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/64cbef27535643b585b0b3f8b02c6573] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=35.5 K 2024-12-05T07:40:06,203 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/240c30feed4a40fb994baad76191adba, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a82e7fe6410942888a6782564442d564, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/e00c8623f92c4b8c89da982592607296] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=37.8 K 2024-12-05T07:40:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:06,204 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:06,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-12-05T07:40:06,204 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f22c6eec3844e91aad875cfcf92e2fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733384403175 2024-12-05T07:40:06,205 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 240c30feed4a40fb994baad76191adba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733384403175 2024-12-05T07:40:06,205 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 753072476fa4424e881a5fd8c5393594, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733384403631 2024-12-05T07:40:06,206 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a82e7fe6410942888a6782564442d564, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733384403631 2024-12-05T07:40:06,206 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 64cbef27535643b585b0b3f8b02c6573, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733384404859 2024-12-05T07:40:06,207 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e00c8623f92c4b8c89da982592607296, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733384404859 2024-12-05T07:40:06,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/f2cba269afb6404eafcd8d09dbd932ea is 50, key is test_row_0/A:col10/1733384406199/Put/seqid=0 2024-12-05T07:40:06,233 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#B#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:06,233 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:06,233 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/3dbef7207fc54b0a9d7b3469094867e0 is 50, key is test_row_0/A:col10/1733384404859/Put/seqid=0 2024-12-05T07:40:06,233 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/41b89a4daa9f49dc8d604d49cf65173d is 50, key is test_row_0/B:col10/1733384404859/Put/seqid=0 2024-12-05T07:40:06,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741864_1040 (size=12409) 2024-12-05T07:40:06,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741863_1039 (size=12409) 2024-12-05T07:40:06,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741865_1041 (size=14541) 2024-12-05T07:40:06,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384466328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384466396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384466406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384466406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384466406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384466436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T07:40:06,467 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:06,467 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:06,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-05T07:40:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T07:40:06,470 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:06,471 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:06,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T07:40:06,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T07:40:06,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:06,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:06,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:06,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:06,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:06,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384466646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,658 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/41b89a4daa9f49dc8d604d49cf65173d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/41b89a4daa9f49dc8d604d49cf65173d 2024-12-05T07:40:06,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/f2cba269afb6404eafcd8d09dbd932ea 2024-12-05T07:40:06,672 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/3dbef7207fc54b0a9d7b3469094867e0 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/3dbef7207fc54b0a9d7b3469094867e0 2024-12-05T07:40:06,673 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/B of 27ae0e4955183fa53f189012e681dac5 into 41b89a4daa9f49dc8d604d49cf65173d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:06,673 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:06,673 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/B, priority=13, startTime=1733384406201; duration=0sec 2024-12-05T07:40:06,673 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:06,673 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:06,674 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:06,676 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:06,677 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/C is initiating minor compaction (all files) 2024-12-05T07:40:06,677 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/C in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:06,677 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/58ce132f8d92435e8f039830cd0bfd4e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/52777ccf3c1847d6a85a12c184b27d2c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ce65850ebcef4f25b248c935040db891, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/80453c9a701c4def9b1f69833bb708b8] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=47.1 K 2024-12-05T07:40:06,678 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 58ce132f8d92435e8f039830cd0bfd4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733384402464 2024-12-05T07:40:06,679 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 52777ccf3c1847d6a85a12c184b27d2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1733384403175 2024-12-05T07:40:06,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/2a2300d27848466086c569b34a660089 is 50, key is test_row_0/B:col10/1733384406199/Put/seqid=0 2024-12-05T07:40:06,683 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting ce65850ebcef4f25b248c935040db891, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1733384403631 2024-12-05T07:40:06,687 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 80453c9a701c4def9b1f69833bb708b8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733384404859 2024-12-05T07:40:06,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741866_1042 (size=12151) 2024-12-05T07:40:06,691 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/A of 27ae0e4955183fa53f189012e681dac5 into 3dbef7207fc54b0a9d7b3469094867e0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:06,691 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:06,691 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/A, priority=13, startTime=1733384406200; duration=0sec 2024-12-05T07:40:06,691 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:06,691 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:06,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/2a2300d27848466086c569b34a660089 2024-12-05T07:40:06,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/1c5ed17aa5c54e41a213742481a4fdbd is 50, key is test_row_0/C:col10/1733384406199/Put/seqid=0 2024-12-05T07:40:06,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384466706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,715 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#C#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:06,715 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/4e09bd4997264eeeba67c7060c165199 is 50, key is test_row_0/C:col10/1733384404859/Put/seqid=0 2024-12-05T07:40:06,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384466716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384466717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384466717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:06,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741867_1043 (size=12151) 2024-12-05T07:40:06,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741868_1044 (size=12375) 2024-12-05T07:40:06,744 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/4e09bd4997264eeeba67c7060c165199 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/4e09bd4997264eeeba67c7060c165199 2024-12-05T07:40:06,755 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/C of 27ae0e4955183fa53f189012e681dac5 into 4e09bd4997264eeeba67c7060c165199(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:06,755 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:06,756 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/C, priority=12, startTime=1733384406201; duration=0sec 2024-12-05T07:40:06,756 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:06,756 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:C 2024-12-05T07:40:06,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T07:40:06,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:06,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:06,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:06,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:06,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T07:40:06,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T07:40:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:06,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:06,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384466956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T07:40:07,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:07,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T07:40:07,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/1c5ed17aa5c54e41a213742481a4fdbd 2024-12-05T07:40:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/f2cba269afb6404eafcd8d09dbd932ea as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f2cba269afb6404eafcd8d09dbd932ea 2024-12-05T07:40:07,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f2cba269afb6404eafcd8d09dbd932ea, entries=200, sequenceid=154, filesize=14.2 K 2024-12-05T07:40:07,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/2a2300d27848466086c569b34a660089 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/2a2300d27848466086c569b34a660089 2024-12-05T07:40:07,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/2a2300d27848466086c569b34a660089, entries=150, sequenceid=154, filesize=11.9 K 2024-12-05T07:40:07,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/1c5ed17aa5c54e41a213742481a4fdbd as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/1c5ed17aa5c54e41a213742481a4fdbd 2024-12-05T07:40:07,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/1c5ed17aa5c54e41a213742481a4fdbd, entries=150, sequenceid=154, filesize=11.9 K 2024-12-05T07:40:07,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 27ae0e4955183fa53f189012e681dac5 in 960ms, sequenceid=154, compaction requested=false 2024-12-05T07:40:07,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:07,228 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:07,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-05T07:40:07,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/420fa540985a4f5cbff22ca64008c18a is 50, key is test_row_0/A:col10/1733384407229/Put/seqid=0 2024-12-05T07:40:07,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741869_1045 (size=14541) 2024-12-05T07:40:07,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T07:40:07,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:07,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384467280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384467282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384467288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384467288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384467397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384467397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T07:40:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:07,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384467397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384467398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384467466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T07:40:07,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:07,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T07:40:07,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384467606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384467607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384467616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384467616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/420fa540985a4f5cbff22ca64008c18a 2024-12-05T07:40:07,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/081621c4f8fd4a5d83a0d75f151ba440 is 50, key is test_row_0/B:col10/1733384407229/Put/seqid=0 2024-12-05T07:40:07,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T07:40:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:07,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741870_1046 (size=12151) 2024-12-05T07:40:07,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/081621c4f8fd4a5d83a0d75f151ba440 2024-12-05T07:40:07,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/08111a377326478d85678865f6ef44dd is 50, key is test_row_0/C:col10/1733384407229/Put/seqid=0 2024-12-05T07:40:07,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741871_1047 (size=12151) 2024-12-05T07:40:07,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/08111a377326478d85678865f6ef44dd 2024-12-05T07:40:07,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/420fa540985a4f5cbff22ca64008c18a as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/420fa540985a4f5cbff22ca64008c18a 2024-12-05T07:40:07,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/420fa540985a4f5cbff22ca64008c18a, entries=200, sequenceid=177, filesize=14.2 K 2024-12-05T07:40:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/081621c4f8fd4a5d83a0d75f151ba440 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/081621c4f8fd4a5d83a0d75f151ba440 2024-12-05T07:40:07,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/081621c4f8fd4a5d83a0d75f151ba440, entries=150, sequenceid=177, filesize=11.9 K 2024-12-05T07:40:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/08111a377326478d85678865f6ef44dd as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/08111a377326478d85678865f6ef44dd 2024-12-05T07:40:07,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/08111a377326478d85678865f6ef44dd, entries=150, sequenceid=177, filesize=11.9 K 2024-12-05T07:40:07,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 27ae0e4955183fa53f189012e681dac5 in 605ms, sequenceid=177, compaction requested=true 2024-12-05T07:40:07,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:07,834 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:07,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:07,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:07,835 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:07,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:07,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:07,836 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:07,836 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:07,836 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/A is initiating minor compaction (all files) 2024-12-05T07:40:07,836 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/B is initiating minor compaction (all files) 2024-12-05T07:40:07,837 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/B in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:07,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:07,837 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/A in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,838 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/3dbef7207fc54b0a9d7b3469094867e0, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f2cba269afb6404eafcd8d09dbd932ea, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/420fa540985a4f5cbff22ca64008c18a] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=40.5 K 2024-12-05T07:40:07,838 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/41b89a4daa9f49dc8d604d49cf65173d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/2a2300d27848466086c569b34a660089, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/081621c4f8fd4a5d83a0d75f151ba440] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=35.9 K 2024-12-05T07:40:07,839 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dbef7207fc54b0a9d7b3469094867e0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733384404859 2024-12-05T07:40:07,839 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2cba269afb6404eafcd8d09dbd932ea, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733384406069 2024-12-05T07:40:07,839 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 41b89a4daa9f49dc8d604d49cf65173d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733384404859 2024-12-05T07:40:07,840 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a2300d27848466086c569b34a660089, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733384406069 2024-12-05T07:40:07,841 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 081621c4f8fd4a5d83a0d75f151ba440, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733384406304 2024-12-05T07:40:07,841 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 420fa540985a4f5cbff22ca64008c18a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733384406304 2024-12-05T07:40:07,860 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#B#compaction#35 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:07,861 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/a93edb60351844d3be06f414f88c75bf is 50, key is test_row_0/B:col10/1733384407229/Put/seqid=0 2024-12-05T07:40:07,865 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#A#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:07,865 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/ee7042d3e0e24fdfa3374cb2dd739943 is 50, key is test_row_0/A:col10/1733384407229/Put/seqid=0 2024-12-05T07:40:07,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T07:40:07,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,878 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-05T07:40:07,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741872_1048 (size=12561) 2024-12-05T07:40:07,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741873_1049 (size=12561) 2024-12-05T07:40:07,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/aeeb151e2f944b4dad6b05f0d582d3fb is 50, key is test_row_0/A:col10/1733384407276/Put/seqid=0 2024-12-05T07:40:07,907 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/ee7042d3e0e24fdfa3374cb2dd739943 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/ee7042d3e0e24fdfa3374cb2dd739943 2024-12-05T07:40:07,917 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/A of 27ae0e4955183fa53f189012e681dac5 into ee7042d3e0e24fdfa3374cb2dd739943(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:07,917 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:07,917 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/A, priority=13, startTime=1733384407834; duration=0sec 2024-12-05T07:40:07,917 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:07,917 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:07,917 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:07,919 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:07,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:07,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:07,919 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/C is initiating minor compaction (all files) 2024-12-05T07:40:07,919 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/C in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:07,919 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/4e09bd4997264eeeba67c7060c165199, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/1c5ed17aa5c54e41a213742481a4fdbd, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/08111a377326478d85678865f6ef44dd] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=35.8 K 2024-12-05T07:40:07,920 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e09bd4997264eeeba67c7060c165199, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733384404859 2024-12-05T07:40:07,921 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c5ed17aa5c54e41a213742481a4fdbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733384406069 2024-12-05T07:40:07,921 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08111a377326478d85678865f6ef44dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733384406304 2024-12-05T07:40:07,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741874_1050 (size=12151) 2024-12-05T07:40:07,923 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/aeeb151e2f944b4dad6b05f0d582d3fb 2024-12-05T07:40:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/70baa525844b43569a1774a1af65e581 is 50, key is test_row_0/B:col10/1733384407276/Put/seqid=0 2024-12-05T07:40:07,953 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#C#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:07,953 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/eadff0b33a4a48d28b4f6c929b24754e is 50, key is test_row_0/C:col10/1733384407229/Put/seqid=0 2024-12-05T07:40:07,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384467957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384467959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384467967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384467968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:07,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741876_1052 (size=12151) 2024-12-05T07:40:07,994 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/70baa525844b43569a1774a1af65e581 2024-12-05T07:40:08,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741875_1051 (size=12527) 2024-12-05T07:40:08,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/ccd846ec4f3e4f85a64509e820da93b4 is 50, key is test_row_0/C:col10/1733384407276/Put/seqid=0 2024-12-05T07:40:08,023 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/eadff0b33a4a48d28b4f6c929b24754e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/eadff0b33a4a48d28b4f6c929b24754e 2024-12-05T07:40:08,035 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/C of 27ae0e4955183fa53f189012e681dac5 into eadff0b33a4a48d28b4f6c929b24754e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:08,035 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:08,035 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/C, priority=13, startTime=1733384407836; duration=0sec 2024-12-05T07:40:08,035 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:08,035 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:C 2024-12-05T07:40:08,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741877_1053 (size=12151) 2024-12-05T07:40:08,047 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/ccd846ec4f3e4f85a64509e820da93b4 2024-12-05T07:40:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/aeeb151e2f944b4dad6b05f0d582d3fb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/aeeb151e2f944b4dad6b05f0d582d3fb 2024-12-05T07:40:08,071 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/aeeb151e2f944b4dad6b05f0d582d3fb, entries=150, sequenceid=194, filesize=11.9 K 2024-12-05T07:40:08,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/70baa525844b43569a1774a1af65e581 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/70baa525844b43569a1774a1af65e581 2024-12-05T07:40:08,085 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/70baa525844b43569a1774a1af65e581, entries=150, sequenceid=194, filesize=11.9 K 2024-12-05T07:40:08,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/ccd846ec4f3e4f85a64509e820da93b4 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ccd846ec4f3e4f85a64509e820da93b4 2024-12-05T07:40:08,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384468077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384468077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384468077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384468077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,098 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ccd846ec4f3e4f85a64509e820da93b4, entries=150, sequenceid=194, filesize=11.9 K 2024-12-05T07:40:08,100 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 27ae0e4955183fa53f189012e681dac5 in 222ms, sequenceid=194, compaction requested=false 2024-12-05T07:40:08,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:08,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-05T07:40:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-05T07:40:08,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-05T07:40:08,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6310 sec 2024-12-05T07:40:08,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.6460 sec 2024-12-05T07:40:08,291 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/a93edb60351844d3be06f414f88c75bf as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/a93edb60351844d3be06f414f88c75bf 2024-12-05T07:40:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:08,299 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:08,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T07:40:08,303 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/B of 27ae0e4955183fa53f189012e681dac5 into a93edb60351844d3be06f414f88c75bf(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:08,303 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:08,303 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/B, priority=13, startTime=1733384407835; duration=0sec 2024-12-05T07:40:08,303 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:08,303 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:08,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/433aaf63ab794a2fac61fd14816bc602 is 50, key is test_row_0/A:col10/1733384408297/Put/seqid=0 2024-12-05T07:40:08,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741878_1054 (size=12151) 2024-12-05T07:40:08,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384468323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384468373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384468373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384468373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384468476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384468486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384468487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384468487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384468487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,493 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 27ae0e4955183fa53f189012e681dac5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T07:40:08,617 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:08,617 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-05T07:40:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T07:40:08,621 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:08,622 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:08,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:08,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384468686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384468696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384468696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384468696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T07:40:08,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/433aaf63ab794a2fac61fd14816bc602 2024-12-05T07:40:08,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/ce5380b2c1d24d0199456a2bc94dd509 is 50, key is test_row_0/B:col10/1733384408297/Put/seqid=0 2024-12-05T07:40:08,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T07:40:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:08,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:08,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:08,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:08,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741879_1055 (size=12151) 2024-12-05T07:40:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T07:40:08,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T07:40:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:08,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:08,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:08,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:08,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:08,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384468996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:09,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384469006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:09,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:09,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384469006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384469006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:09,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T07:40:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/ce5380b2c1d24d0199456a2bc94dd509 2024-12-05T07:40:09,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/8dca70746f2f4a73bcaf0a7d891de2b2 is 50, key is test_row_0/C:col10/1733384408297/Put/seqid=0 2024-12-05T07:40:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T07:40:09,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T07:40:09,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:09,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741880_1056 (size=12151) 2024-12-05T07:40:09,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T07:40:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384469506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:09,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:09,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:09,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384469516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384469516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384469516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:09,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:09,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T07:40:09,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:09,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/8dca70746f2f4a73bcaf0a7d891de2b2 2024-12-05T07:40:09,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/433aaf63ab794a2fac61fd14816bc602 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/433aaf63ab794a2fac61fd14816bc602 2024-12-05T07:40:09,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/433aaf63ab794a2fac61fd14816bc602, entries=150, sequenceid=217, filesize=11.9 K 2024-12-05T07:40:09,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/ce5380b2c1d24d0199456a2bc94dd509 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/ce5380b2c1d24d0199456a2bc94dd509 2024-12-05T07:40:09,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/ce5380b2c1d24d0199456a2bc94dd509, entries=150, sequenceid=217, filesize=11.9 K 2024-12-05T07:40:09,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T07:40:09,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:09,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/8dca70746f2f4a73bcaf0a7d891de2b2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8dca70746f2f4a73bcaf0a7d891de2b2 2024-12-05T07:40:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:09,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8dca70746f2f4a73bcaf0a7d891de2b2, entries=150, sequenceid=217, filesize=11.9 K 2024-12-05T07:40:09,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 27ae0e4955183fa53f189012e681dac5 in 1436ms, sequenceid=217, compaction requested=true 2024-12-05T07:40:09,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:09,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:09,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:09,735 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:09,735 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:09,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:09,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:09,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:09,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:09,737 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:09,737 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/A is initiating minor compaction (all files) 2024-12-05T07:40:09,737 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/A in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,737 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/ee7042d3e0e24fdfa3374cb2dd739943, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/aeeb151e2f944b4dad6b05f0d582d3fb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/433aaf63ab794a2fac61fd14816bc602] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=36.0 K 2024-12-05T07:40:09,738 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:09,738 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/B is initiating minor compaction (all files) 2024-12-05T07:40:09,738 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/B in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,738 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/a93edb60351844d3be06f414f88c75bf, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/70baa525844b43569a1774a1af65e581, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/ce5380b2c1d24d0199456a2bc94dd509] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=36.0 K 2024-12-05T07:40:09,740 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting ee7042d3e0e24fdfa3374cb2dd739943, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733384406304 2024-12-05T07:40:09,741 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a93edb60351844d3be06f414f88c75bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733384406304 2024-12-05T07:40:09,741 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting aeeb151e2f944b4dad6b05f0d582d3fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733384407263 2024-12-05T07:40:09,741 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70baa525844b43569a1774a1af65e581, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733384407263 2024-12-05T07:40:09,741 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 433aaf63ab794a2fac61fd14816bc602, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733384407962 2024-12-05T07:40:09,742 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce5380b2c1d24d0199456a2bc94dd509, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733384407962 2024-12-05T07:40:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T07:40:09,770 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#B#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:09,771 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/cf0735684e2e428e919993917a4a278f is 50, key is test_row_0/B:col10/1733384408297/Put/seqid=0 2024-12-05T07:40:09,779 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#A#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:09,780 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/93f9bb8192d54e7286c00e4bbd68a537 is 50, key is test_row_0/A:col10/1733384408297/Put/seqid=0 2024-12-05T07:40:09,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741882_1058 (size=12663) 2024-12-05T07:40:09,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741881_1057 (size=12663) 2024-12-05T07:40:09,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T07:40:09,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:09,877 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-12-05T07:40:09,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/63b8f3b303c146159855927e1669fa4e is 50, key is test_row_0/A:col10/1733384408318/Put/seqid=0 2024-12-05T07:40:09,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741883_1059 (size=12151) 2024-12-05T07:40:10,216 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/93f9bb8192d54e7286c00e4bbd68a537 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/93f9bb8192d54e7286c00e4bbd68a537 2024-12-05T07:40:10,227 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/cf0735684e2e428e919993917a4a278f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/cf0735684e2e428e919993917a4a278f 2024-12-05T07:40:10,245 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/B of 27ae0e4955183fa53f189012e681dac5 into cf0735684e2e428e919993917a4a278f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:10,245 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:10,245 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/B, priority=13, startTime=1733384409735; duration=0sec 2024-12-05T07:40:10,245 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:10,245 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:10,245 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:10,247 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/A of 27ae0e4955183fa53f189012e681dac5 into 93f9bb8192d54e7286c00e4bbd68a537(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:10,247 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:10,247 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/A, priority=13, startTime=1733384409735; duration=0sec 2024-12-05T07:40:10,247 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:10,247 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:10,248 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:10,248 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/C is initiating minor compaction (all files) 2024-12-05T07:40:10,248 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/C in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:10,249 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/eadff0b33a4a48d28b4f6c929b24754e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ccd846ec4f3e4f85a64509e820da93b4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8dca70746f2f4a73bcaf0a7d891de2b2] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=36.0 K 2024-12-05T07:40:10,249 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting eadff0b33a4a48d28b4f6c929b24754e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733384406304 2024-12-05T07:40:10,250 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccd846ec4f3e4f85a64509e820da93b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733384407263 2024-12-05T07:40:10,251 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dca70746f2f4a73bcaf0a7d891de2b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733384407962 2024-12-05T07:40:10,278 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#C#compaction#47 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:10,279 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/a8cf4fad05644eb682de19e6db6a557e is 50, key is test_row_0/C:col10/1733384408297/Put/seqid=0 2024-12-05T07:40:10,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741884_1060 (size=12629) 2024-12-05T07:40:10,346 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/63b8f3b303c146159855927e1669fa4e 2024-12-05T07:40:10,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/b9836cadbc4e400da4793ab46a3d75b7 is 50, key is test_row_0/B:col10/1733384408318/Put/seqid=0 2024-12-05T07:40:10,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741885_1061 (size=12151) 2024-12-05T07:40:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:10,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:10,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384470544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384470545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384470548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384470550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384470552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384470657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384470666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384470666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384470666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384470667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,706 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/a8cf4fad05644eb682de19e6db6a557e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a8cf4fad05644eb682de19e6db6a557e 2024-12-05T07:40:10,716 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/C of 27ae0e4955183fa53f189012e681dac5 into a8cf4fad05644eb682de19e6db6a557e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:10,716 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:10,716 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/C, priority=13, startTime=1733384409736; duration=0sec 2024-12-05T07:40:10,716 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:10,716 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:C 2024-12-05T07:40:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T07:40:10,796 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/b9836cadbc4e400da4793ab46a3d75b7 2024-12-05T07:40:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/3c44b8f4d3204892908e7d4a8ec53545 is 50, key is test_row_0/C:col10/1733384408318/Put/seqid=0 2024-12-05T07:40:10,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384470866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384470876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384470877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384470878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384470877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:10,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741886_1062 (size=12151) 2024-12-05T07:40:11,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384471186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384471196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384471196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384471196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384471196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,301 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/3c44b8f4d3204892908e7d4a8ec53545 2024-12-05T07:40:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/63b8f3b303c146159855927e1669fa4e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/63b8f3b303c146159855927e1669fa4e 2024-12-05T07:40:11,322 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/63b8f3b303c146159855927e1669fa4e, entries=150, sequenceid=234, filesize=11.9 K 2024-12-05T07:40:11,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/b9836cadbc4e400da4793ab46a3d75b7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b9836cadbc4e400da4793ab46a3d75b7 2024-12-05T07:40:11,335 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b9836cadbc4e400da4793ab46a3d75b7, entries=150, sequenceid=234, filesize=11.9 K 2024-12-05T07:40:11,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/3c44b8f4d3204892908e7d4a8ec53545 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/3c44b8f4d3204892908e7d4a8ec53545 2024-12-05T07:40:11,348 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/3c44b8f4d3204892908e7d4a8ec53545, entries=150, sequenceid=234, filesize=11.9 K 2024-12-05T07:40:11,363 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 27ae0e4955183fa53f189012e681dac5 in 1486ms, sequenceid=234, compaction requested=false 2024-12-05T07:40:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-05T07:40:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-05T07:40:11,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-05T07:40:11,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7480 sec 2024-12-05T07:40:11,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.7630 sec 2024-12-05T07:40:11,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:11,702 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:11,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T07:40:11,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/06159bdd3b62434fb87041620543c254 is 50, key is test_row_0/A:col10/1733384411698/Put/seqid=0 2024-12-05T07:40:11,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741887_1063 (size=14591) 2024-12-05T07:40:11,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/06159bdd3b62434fb87041620543c254 2024-12-05T07:40:11,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/d4ef570606234b8ab210cb9196c40d81 is 50, key is test_row_0/B:col10/1733384411698/Put/seqid=0 2024-12-05T07:40:11,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741888_1064 (size=12201) 2024-12-05T07:40:11,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/d4ef570606234b8ab210cb9196c40d81 2024-12-05T07:40:11,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384471800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384471800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384471802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/4764f65c0af04455989f441a56465370 is 50, key is test_row_0/C:col10/1733384411698/Put/seqid=0 2024-12-05T07:40:11,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384471804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384471805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741889_1065 (size=12201) 2024-12-05T07:40:11,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/4764f65c0af04455989f441a56465370 2024-12-05T07:40:11,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/06159bdd3b62434fb87041620543c254 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/06159bdd3b62434fb87041620543c254 2024-12-05T07:40:11,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/06159bdd3b62434fb87041620543c254, entries=200, sequenceid=258, filesize=14.2 K 2024-12-05T07:40:11,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/d4ef570606234b8ab210cb9196c40d81 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/d4ef570606234b8ab210cb9196c40d81 2024-12-05T07:40:11,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/d4ef570606234b8ab210cb9196c40d81, entries=150, sequenceid=258, filesize=11.9 K 2024-12-05T07:40:11,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/4764f65c0af04455989f441a56465370 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/4764f65c0af04455989f441a56465370 2024-12-05T07:40:11,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/4764f65c0af04455989f441a56465370, entries=150, sequenceid=258, filesize=11.9 K 2024-12-05T07:40:11,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 27ae0e4955183fa53f189012e681dac5 in 208ms, sequenceid=258, compaction requested=true 2024-12-05T07:40:11,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:11,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:11,911 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:11,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:11,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:11,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:11,911 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:11,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:11,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:11,912 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:11,912 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/B is initiating minor compaction (all files) 2024-12-05T07:40:11,913 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/B in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:11,913 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/cf0735684e2e428e919993917a4a278f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b9836cadbc4e400da4793ab46a3d75b7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/d4ef570606234b8ab210cb9196c40d81] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=36.1 K 2024-12-05T07:40:11,913 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:11,913 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/A is initiating minor compaction (all files) 2024-12-05T07:40:11,913 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/A in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:11,913 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/93f9bb8192d54e7286c00e4bbd68a537, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/63b8f3b303c146159855927e1669fa4e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/06159bdd3b62434fb87041620543c254] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=38.5 K 2024-12-05T07:40:11,914 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f9bb8192d54e7286c00e4bbd68a537, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733384407962 2024-12-05T07:40:11,914 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting cf0735684e2e428e919993917a4a278f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733384407962 2024-12-05T07:40:11,915 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63b8f3b303c146159855927e1669fa4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733384408318 2024-12-05T07:40:11,916 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting b9836cadbc4e400da4793ab46a3d75b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733384408318 2024-12-05T07:40:11,916 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06159bdd3b62434fb87041620543c254, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733384410540 2024-12-05T07:40:11,917 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting d4ef570606234b8ab210cb9196c40d81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733384410548 2024-12-05T07:40:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:11,924 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:11,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-05T07:40:11,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/abacc1b2dfff47e3828a7ddf26a90814 is 50, key is test_row_0/A:col10/1733384411800/Put/seqid=0 2024-12-05T07:40:11,956 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#B#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:11,957 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/08ca77aa0ced4bb1a2c58a552b20ac83 is 50, key is test_row_0/B:col10/1733384411698/Put/seqid=0 2024-12-05T07:40:11,971 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#A#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:11,972 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/75935a6dacae40ccbc0151de3aff62bd is 50, key is test_row_0/A:col10/1733384411698/Put/seqid=0 2024-12-05T07:40:11,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384471963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384471966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384471969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384471973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:11,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384471974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:11,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741891_1067 (size=12815) 2024-12-05T07:40:12,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741890_1066 (size=12301) 2024-12-05T07:40:12,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/abacc1b2dfff47e3828a7ddf26a90814 2024-12-05T07:40:12,006 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/08ca77aa0ced4bb1a2c58a552b20ac83 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/08ca77aa0ced4bb1a2c58a552b20ac83 2024-12-05T07:40:12,017 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/B of 27ae0e4955183fa53f189012e681dac5 into 08ca77aa0ced4bb1a2c58a552b20ac83(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:12,017 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:12,017 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/B, priority=13, startTime=1733384411911; duration=0sec 2024-12-05T07:40:12,017 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:12,017 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:12,017 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:12,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741892_1068 (size=12815) 2024-12-05T07:40:12,019 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36981 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:12,019 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/C is initiating minor compaction (all files) 2024-12-05T07:40:12,019 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/C in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:12,020 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a8cf4fad05644eb682de19e6db6a557e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/3c44b8f4d3204892908e7d4a8ec53545, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/4764f65c0af04455989f441a56465370] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=36.1 K 2024-12-05T07:40:12,020 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a8cf4fad05644eb682de19e6db6a557e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733384407962 2024-12-05T07:40:12,020 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c44b8f4d3204892908e7d4a8ec53545, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733384408318 2024-12-05T07:40:12,021 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4764f65c0af04455989f441a56465370, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733384410548 2024-12-05T07:40:12,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/5ce4ef2f22774cbcad2fde7e6c9c45f6 is 50, key is test_row_0/B:col10/1733384411800/Put/seqid=0 2024-12-05T07:40:12,034 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#C#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:12,035 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/21d0be3fd4194585ae2f82b26902b7b2 is 50, key is test_row_0/C:col10/1733384411698/Put/seqid=0 2024-12-05T07:40:12,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741893_1069 (size=12301) 2024-12-05T07:40:12,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741894_1070 (size=12781) 2024-12-05T07:40:12,048 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/21d0be3fd4194585ae2f82b26902b7b2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/21d0be3fd4194585ae2f82b26902b7b2 2024-12-05T07:40:12,055 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/C of 27ae0e4955183fa53f189012e681dac5 into 21d0be3fd4194585ae2f82b26902b7b2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:12,056 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:12,056 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/C, priority=13, startTime=1733384411911; duration=0sec 2024-12-05T07:40:12,056 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:12,056 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:C 2024-12-05T07:40:12,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384472075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384472086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384472087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384472087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384472088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384472286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384472296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384472297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384472297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384472302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,426 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/75935a6dacae40ccbc0151de3aff62bd as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/75935a6dacae40ccbc0151de3aff62bd 2024-12-05T07:40:12,435 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/A of 27ae0e4955183fa53f189012e681dac5 into 75935a6dacae40ccbc0151de3aff62bd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:12,436 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:12,436 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/A, priority=13, startTime=1733384411910; duration=0sec 2024-12-05T07:40:12,436 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:12,436 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:12,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/5ce4ef2f22774cbcad2fde7e6c9c45f6 2024-12-05T07:40:12,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/a512d5e071af422a83d238d31e5c5f29 is 50, key is test_row_0/C:col10/1733384411800/Put/seqid=0 2024-12-05T07:40:12,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741895_1071 (size=12301) 2024-12-05T07:40:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384472595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384472606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384472606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384472617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384472617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T07:40:12,789 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:12,790 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-05T07:40:12,794 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T07:40:12,796 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:12,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:12,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/a512d5e071af422a83d238d31e5c5f29 2024-12-05T07:40:12,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/abacc1b2dfff47e3828a7ddf26a90814 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/abacc1b2dfff47e3828a7ddf26a90814 2024-12-05T07:40:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T07:40:12,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/abacc1b2dfff47e3828a7ddf26a90814, entries=150, sequenceid=276, filesize=12.0 K 2024-12-05T07:40:12,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/5ce4ef2f22774cbcad2fde7e6c9c45f6 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/5ce4ef2f22774cbcad2fde7e6c9c45f6 2024-12-05T07:40:12,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/5ce4ef2f22774cbcad2fde7e6c9c45f6, entries=150, sequenceid=276, filesize=12.0 K 2024-12-05T07:40:12,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/a512d5e071af422a83d238d31e5c5f29 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a512d5e071af422a83d238d31e5c5f29 2024-12-05T07:40:12,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a512d5e071af422a83d238d31e5c5f29, entries=150, sequenceid=276, filesize=12.0 K 2024-12-05T07:40:12,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 27ae0e4955183fa53f189012e681dac5 in 1010ms, sequenceid=276, compaction requested=false 2024-12-05T07:40:12,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:12,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T07:40:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:12,953 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-12-05T07:40:12,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/81ebc80656594cf081200912aa1ca419 is 50, key is test_row_0/A:col10/1733384411963/Put/seqid=0 2024-12-05T07:40:13,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741896_1072 (size=9857) 2024-12-05T07:40:13,013 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/81ebc80656594cf081200912aa1ca419 2024-12-05T07:40:13,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/767449440e4346ca9b93c7c7fa121f84 is 50, key is test_row_0/B:col10/1733384411963/Put/seqid=0 2024-12-05T07:40:13,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741897_1073 (size=9857) 2024-12-05T07:40:13,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:13,111 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/767449440e4346ca9b93c7c7fa121f84 2024-12-05T07:40:13,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:13,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T07:40:13,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/e2c5fd4e5c464cc8ae863f057066ab78 is 50, key is test_row_0/C:col10/1733384411963/Put/seqid=0 2024-12-05T07:40:13,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384473143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384473145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384473145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384473147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384473156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741898_1074 (size=9857) 2024-12-05T07:40:13,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384473266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384473266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384473266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384473267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384473274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T07:40:13,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384473476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384473476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384473476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384473476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384473485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,596 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/e2c5fd4e5c464cc8ae863f057066ab78 2024-12-05T07:40:13,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/81ebc80656594cf081200912aa1ca419 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/81ebc80656594cf081200912aa1ca419 2024-12-05T07:40:13,624 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/81ebc80656594cf081200912aa1ca419, entries=100, sequenceid=297, filesize=9.6 K 2024-12-05T07:40:13,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/767449440e4346ca9b93c7c7fa121f84 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/767449440e4346ca9b93c7c7fa121f84 2024-12-05T07:40:13,637 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/767449440e4346ca9b93c7c7fa121f84, entries=100, sequenceid=297, filesize=9.6 K 2024-12-05T07:40:13,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/e2c5fd4e5c464cc8ae863f057066ab78 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/e2c5fd4e5c464cc8ae863f057066ab78 2024-12-05T07:40:13,648 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/e2c5fd4e5c464cc8ae863f057066ab78, entries=100, sequenceid=297, filesize=9.6 K 2024-12-05T07:40:13,650 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 27ae0e4955183fa53f189012e681dac5 in 698ms, sequenceid=297, compaction requested=true 2024-12-05T07:40:13,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:13,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:13,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-05T07:40:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-05T07:40:13,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-05T07:40:13,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 859 msec 2024-12-05T07:40:13,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 872 msec 2024-12-05T07:40:13,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:13,791 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:13,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-12-05T07:40:13,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/7573284aa2e54b7aad113ab87e6f301e is 50, key is test_row_0/A:col10/1733384413789/Put/seqid=0 2024-12-05T07:40:13,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741899_1075 (size=14741) 2024-12-05T07:40:13,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/7573284aa2e54b7aad113ab87e6f301e 2024-12-05T07:40:13,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/06e9931838ae47dab57d7153943ce9d7 is 50, key is test_row_0/B:col10/1733384413789/Put/seqid=0 2024-12-05T07:40:13,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741900_1076 (size=12301) 2024-12-05T07:40:13,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384473819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384473822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384473868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384473871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384473876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T07:40:13,936 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:13,937 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-05T07:40:13,941 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:13,943 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:13,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T07:40:13,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384473977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384473979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384473981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384473986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384473986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:13,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:13,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:13,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T07:40:14,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T07:40:14,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:14,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384474186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384474186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:14,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384474196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:14,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384474196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:14,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384474197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:14,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T07:40:14,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:14,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/06e9931838ae47dab57d7153943ce9d7 2024-12-05T07:40:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T07:40:14,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/c68c7acd46024fe39620067460e84fa6 is 50, key is test_row_0/C:col10/1733384413789/Put/seqid=0 2024-12-05T07:40:14,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741901_1077 (size=12301) 2024-12-05T07:40:14,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T07:40:14,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:14,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384474496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384474496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:14,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384474505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:14,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:14,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384474506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:14,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384474506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:14,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:14,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T07:40:14,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:14,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T07:40:14,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/c68c7acd46024fe39620067460e84fa6 2024-12-05T07:40:14,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/7573284aa2e54b7aad113ab87e6f301e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/7573284aa2e54b7aad113ab87e6f301e 2024-12-05T07:40:14,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/7573284aa2e54b7aad113ab87e6f301e, entries=200, sequenceid=316, filesize=14.4 K 2024-12-05T07:40:14,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T07:40:14,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:14,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:14,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/06e9931838ae47dab57d7153943ce9d7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/06e9931838ae47dab57d7153943ce9d7 2024-12-05T07:40:14,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/06e9931838ae47dab57d7153943ce9d7, entries=150, sequenceid=316, filesize=12.0 K 2024-12-05T07:40:14,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/c68c7acd46024fe39620067460e84fa6 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c68c7acd46024fe39620067460e84fa6 2024-12-05T07:40:14,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c68c7acd46024fe39620067460e84fa6, entries=150, sequenceid=316, filesize=12.0 K 2024-12-05T07:40:14,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 27ae0e4955183fa53f189012e681dac5 in 954ms, sequenceid=316, compaction requested=true 2024-12-05T07:40:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:14,745 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:14,745 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:14,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:14,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:14,748 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47274 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:14,748 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:14,748 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/A is initiating minor compaction (all files) 2024-12-05T07:40:14,748 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/B is initiating minor compaction (all files) 2024-12-05T07:40:14,748 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/B in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,748 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/A in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,748 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/08ca77aa0ced4bb1a2c58a552b20ac83, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/5ce4ef2f22774cbcad2fde7e6c9c45f6, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/767449440e4346ca9b93c7c7fa121f84, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/06e9931838ae47dab57d7153943ce9d7] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=46.2 K 2024-12-05T07:40:14,748 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/75935a6dacae40ccbc0151de3aff62bd, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/abacc1b2dfff47e3828a7ddf26a90814, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/81ebc80656594cf081200912aa1ca419, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/7573284aa2e54b7aad113ab87e6f301e] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=48.5 K 2024-12-05T07:40:14,748 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75935a6dacae40ccbc0151de3aff62bd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733384410548 2024-12-05T07:40:14,748 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ca77aa0ced4bb1a2c58a552b20ac83, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733384410548 2024-12-05T07:40:14,749 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ce4ef2f22774cbcad2fde7e6c9c45f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733384411793 2024-12-05T07:40:14,749 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting abacc1b2dfff47e3828a7ddf26a90814, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733384411793 2024-12-05T07:40:14,750 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 767449440e4346ca9b93c7c7fa121f84, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733384411963 2024-12-05T07:40:14,750 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ebc80656594cf081200912aa1ca419, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733384411963 2024-12-05T07:40:14,750 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 06e9931838ae47dab57d7153943ce9d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733384413142 2024-12-05T07:40:14,750 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7573284aa2e54b7aad113ab87e6f301e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733384413142 2024-12-05T07:40:14,773 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#B#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:14,774 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/ff0b562f8c024d3898b6288d365b154e is 50, key is test_row_0/B:col10/1733384413789/Put/seqid=0 2024-12-05T07:40:14,777 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:14,777 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/06a18e99dca74871bd563385589ee16f is 50, key is test_row_0/A:col10/1733384413789/Put/seqid=0 2024-12-05T07:40:14,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741903_1079 (size=13051) 2024-12-05T07:40:14,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741902_1078 (size=13051) 2024-12-05T07:40:14,858 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/06a18e99dca74871bd563385589ee16f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/06a18e99dca74871bd563385589ee16f 2024-12-05T07:40:14,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T07:40:14,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,870 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-12-05T07:40:14,874 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/A of 27ae0e4955183fa53f189012e681dac5 into 06a18e99dca74871bd563385589ee16f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:14,874 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:14,874 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/A, priority=12, startTime=1733384414745; duration=0sec 2024-12-05T07:40:14,874 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:14,874 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:14,875 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:14,878 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47240 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:14,878 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/C is initiating minor compaction (all files) 2024-12-05T07:40:14,878 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/C in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:14,878 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/21d0be3fd4194585ae2f82b26902b7b2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a512d5e071af422a83d238d31e5c5f29, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/e2c5fd4e5c464cc8ae863f057066ab78, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c68c7acd46024fe39620067460e84fa6] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=46.1 K 2024-12-05T07:40:14,879 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21d0be3fd4194585ae2f82b26902b7b2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733384410548 2024-12-05T07:40:14,879 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a512d5e071af422a83d238d31e5c5f29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733384411793 2024-12-05T07:40:14,880 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2c5fd4e5c464cc8ae863f057066ab78, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1733384411963 2024-12-05T07:40:14,880 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c68c7acd46024fe39620067460e84fa6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733384413142 2024-12-05T07:40:14,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/a7ee7d768b5542b6b326ce87ace12164 is 50, key is test_row_0/A:col10/1733384413830/Put/seqid=0 2024-12-05T07:40:14,901 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:14,902 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/2fad64e71407482596820dc121f75950 is 50, key is test_row_0/C:col10/1733384413789/Put/seqid=0 2024-12-05T07:40:14,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741904_1080 (size=12301) 2024-12-05T07:40:14,933 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/a7ee7d768b5542b6b326ce87ace12164 2024-12-05T07:40:14,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/b461021ab40c42fda69b2743e1b516bb is 50, key is test_row_0/B:col10/1733384413830/Put/seqid=0 2024-12-05T07:40:14,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741905_1081 (size=13017) 2024-12-05T07:40:14,988 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/2fad64e71407482596820dc121f75950 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/2fad64e71407482596820dc121f75950 2024-12-05T07:40:15,006 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/C of 27ae0e4955183fa53f189012e681dac5 into 2fad64e71407482596820dc121f75950(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:15,006 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:15,007 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/C, priority=12, startTime=1733384414745; duration=0sec 2024-12-05T07:40:15,007 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:15,007 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:C 2024-12-05T07:40:15,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741906_1082 (size=12301) 2024-12-05T07:40:15,013 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/b461021ab40c42fda69b2743e1b516bb 2024-12-05T07:40:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:15,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/8f2cf6ef951e4c05959f2e0bde915c90 is 50, key is test_row_0/C:col10/1733384413830/Put/seqid=0 2024-12-05T07:40:15,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384475051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384475051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384475053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384475055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384475055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741907_1083 (size=12301) 2024-12-05T07:40:15,074 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/8f2cf6ef951e4c05959f2e0bde915c90 2024-12-05T07:40:15,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/a7ee7d768b5542b6b326ce87ace12164 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a7ee7d768b5542b6b326ce87ace12164 2024-12-05T07:40:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T07:40:15,095 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a7ee7d768b5542b6b326ce87ace12164, entries=150, sequenceid=334, filesize=12.0 K 2024-12-05T07:40:15,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/b461021ab40c42fda69b2743e1b516bb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b461021ab40c42fda69b2743e1b516bb 2024-12-05T07:40:15,121 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b461021ab40c42fda69b2743e1b516bb, entries=150, sequenceid=334, filesize=12.0 K 2024-12-05T07:40:15,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/8f2cf6ef951e4c05959f2e0bde915c90 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8f2cf6ef951e4c05959f2e0bde915c90 2024-12-05T07:40:15,132 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8f2cf6ef951e4c05959f2e0bde915c90, entries=150, sequenceid=334, filesize=12.0 K 2024-12-05T07:40:15,134 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 27ae0e4955183fa53f189012e681dac5 in 263ms, sequenceid=334, compaction requested=false 2024-12-05T07:40:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-05T07:40:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-05T07:40:15,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-05T07:40:15,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1940 sec 2024-12-05T07:40:15,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2050 sec 2024-12-05T07:40:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:15,171 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:15,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T07:40:15,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/f5de23a7da0e4893aab3937aba707ee4 is 50, key is test_row_0/A:col10/1733384415051/Put/seqid=0 2024-12-05T07:40:15,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384475186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384475187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384475190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384475192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384475192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741908_1084 (size=14741) 2024-12-05T07:40:15,250 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/ff0b562f8c024d3898b6288d365b154e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/ff0b562f8c024d3898b6288d365b154e 2024-12-05T07:40:15,260 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/B of 27ae0e4955183fa53f189012e681dac5 into ff0b562f8c024d3898b6288d365b154e(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:15,260 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:15,260 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/B, priority=12, startTime=1733384414745; duration=0sec 2024-12-05T07:40:15,261 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:15,261 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:15,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384475306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384475306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384475306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384475306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384475306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384475516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384475517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384475517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384475517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384475526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/f5de23a7da0e4893aab3937aba707ee4 2024-12-05T07:40:15,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/c439b7de64c44a0cb547103fd071820f is 50, key is test_row_0/B:col10/1733384415051/Put/seqid=0 2024-12-05T07:40:15,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741909_1085 (size=12301) 2024-12-05T07:40:15,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/c439b7de64c44a0cb547103fd071820f 2024-12-05T07:40:15,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/9cb6916699474e35be6a652e5345323a is 50, key is test_row_0/C:col10/1733384415051/Put/seqid=0 2024-12-05T07:40:15,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741910_1086 (size=12301) 2024-12-05T07:40:15,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/9cb6916699474e35be6a652e5345323a 2024-12-05T07:40:15,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/f5de23a7da0e4893aab3937aba707ee4 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f5de23a7da0e4893aab3937aba707ee4 2024-12-05T07:40:15,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f5de23a7da0e4893aab3937aba707ee4, entries=200, sequenceid=359, filesize=14.4 K 2024-12-05T07:40:15,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/c439b7de64c44a0cb547103fd071820f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c439b7de64c44a0cb547103fd071820f 2024-12-05T07:40:15,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c439b7de64c44a0cb547103fd071820f, entries=150, sequenceid=359, filesize=12.0 K 2024-12-05T07:40:15,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/9cb6916699474e35be6a652e5345323a as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/9cb6916699474e35be6a652e5345323a 2024-12-05T07:40:15,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/9cb6916699474e35be6a652e5345323a, entries=150, sequenceid=359, filesize=12.0 K 2024-12-05T07:40:15,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 27ae0e4955183fa53f189012e681dac5 in 609ms, sequenceid=359, compaction requested=true 2024-12-05T07:40:15,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:15,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:15,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:15,782 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:15,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:15,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:15,782 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:15,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:15,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:15,784 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:15,784 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:15,784 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/B is initiating minor compaction (all files) 2024-12-05T07:40:15,784 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/A is initiating minor compaction (all files) 2024-12-05T07:40:15,784 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/B in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:15,784 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/A in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:15,784 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/ff0b562f8c024d3898b6288d365b154e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b461021ab40c42fda69b2743e1b516bb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c439b7de64c44a0cb547103fd071820f] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=36.8 K 2024-12-05T07:40:15,784 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/06a18e99dca74871bd563385589ee16f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a7ee7d768b5542b6b326ce87ace12164, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f5de23a7da0e4893aab3937aba707ee4] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=39.2 K 2024-12-05T07:40:15,785 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting ff0b562f8c024d3898b6288d365b154e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733384413142 2024-12-05T07:40:15,785 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06a18e99dca74871bd563385589ee16f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733384413142 2024-12-05T07:40:15,787 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting b461021ab40c42fda69b2743e1b516bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733384413815 2024-12-05T07:40:15,787 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7ee7d768b5542b6b326ce87ace12164, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733384413815 2024-12-05T07:40:15,787 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5de23a7da0e4893aab3937aba707ee4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733384415051 2024-12-05T07:40:15,788 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting c439b7de64c44a0cb547103fd071820f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733384415051 2024-12-05T07:40:15,808 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#A#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:15,809 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/a467398e5ac84553b873104f7d3024e3 is 50, key is test_row_0/A:col10/1733384415051/Put/seqid=0 2024-12-05T07:40:15,810 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:15,811 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/c5fa6e49c1254bd4a3651ffcd2c2ab3c is 50, key is test_row_0/B:col10/1733384415051/Put/seqid=0 2024-12-05T07:40:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:15,831 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:15,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-05T07:40:15,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/dab7854c0ad94a3982f09366abfafa3e is 50, key is test_row_0/A:col10/1733384415190/Put/seqid=0 2024-12-05T07:40:15,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741912_1088 (size=13153) 2024-12-05T07:40:15,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741911_1087 (size=13153) 2024-12-05T07:40:15,873 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/c5fa6e49c1254bd4a3651ffcd2c2ab3c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c5fa6e49c1254bd4a3651ffcd2c2ab3c 2024-12-05T07:40:15,879 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/a467398e5ac84553b873104f7d3024e3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a467398e5ac84553b873104f7d3024e3 2024-12-05T07:40:15,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741913_1089 (size=17181) 2024-12-05T07:40:15,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/dab7854c0ad94a3982f09366abfafa3e 2024-12-05T07:40:15,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384475870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384475870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384475871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384475875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384475882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,888 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/B of 27ae0e4955183fa53f189012e681dac5 into c5fa6e49c1254bd4a3651ffcd2c2ab3c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:15,888 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:15,888 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/B, priority=13, startTime=1733384415782; duration=0sec 2024-12-05T07:40:15,889 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:15,889 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:15,889 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:15,890 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/A of 27ae0e4955183fa53f189012e681dac5 into a467398e5ac84553b873104f7d3024e3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:15,890 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:15,890 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/A, priority=13, startTime=1733384415782; duration=0sec 2024-12-05T07:40:15,890 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:15,890 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:15,890 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:15,890 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/C is initiating minor compaction (all files) 2024-12-05T07:40:15,890 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/C in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:15,890 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/2fad64e71407482596820dc121f75950, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8f2cf6ef951e4c05959f2e0bde915c90, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/9cb6916699474e35be6a652e5345323a] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=36.7 K 2024-12-05T07:40:15,892 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fad64e71407482596820dc121f75950, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733384413142 2024-12-05T07:40:15,893 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f2cf6ef951e4c05959f2e0bde915c90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733384413815 2024-12-05T07:40:15,893 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb6916699474e35be6a652e5345323a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733384415051 2024-12-05T07:40:15,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/7f0b1e1f6d8844dbabeb6128b661214a is 50, key is test_row_0/B:col10/1733384415190/Put/seqid=0 2024-12-05T07:40:15,932 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:15,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741914_1090 (size=12301) 2024-12-05T07:40:15,933 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/3c7e7f8ea34e40b8b8cc0aa623c3fcb1 is 50, key is test_row_0/C:col10/1733384415051/Put/seqid=0 2024-12-05T07:40:15,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741915_1091 (size=13119) 2024-12-05T07:40:15,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384475986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384475986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384475986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384475995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:15,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384475997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T07:40:16,096 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:16,096 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-05T07:40:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T07:40:16,098 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:16,098 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:16,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:16,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384476197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384476198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384476198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T07:40:16,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384476206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384476207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T07:40:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:16,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:16,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/7f0b1e1f6d8844dbabeb6128b661214a 2024-12-05T07:40:16,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/60a7ccdf8a6c43e790c1ebe2c515d678 is 50, key is test_row_0/C:col10/1733384415190/Put/seqid=0 2024-12-05T07:40:16,352 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/3c7e7f8ea34e40b8b8cc0aa623c3fcb1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/3c7e7f8ea34e40b8b8cc0aa623c3fcb1 2024-12-05T07:40:16,362 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/C of 27ae0e4955183fa53f189012e681dac5 into 3c7e7f8ea34e40b8b8cc0aa623c3fcb1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:16,362 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:16,363 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/C, priority=13, startTime=1733384415782; duration=0sec 2024-12-05T07:40:16,363 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:16,363 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:C 2024-12-05T07:40:16,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741916_1092 (size=12301) 2024-12-05T07:40:16,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/60a7ccdf8a6c43e790c1ebe2c515d678 2024-12-05T07:40:16,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/dab7854c0ad94a3982f09366abfafa3e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/dab7854c0ad94a3982f09366abfafa3e 2024-12-05T07:40:16,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/dab7854c0ad94a3982f09366abfafa3e, entries=250, sequenceid=375, filesize=16.8 K 2024-12-05T07:40:16,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/7f0b1e1f6d8844dbabeb6128b661214a as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/7f0b1e1f6d8844dbabeb6128b661214a 2024-12-05T07:40:16,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T07:40:16,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:16,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:16,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:16,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/7f0b1e1f6d8844dbabeb6128b661214a, entries=150, sequenceid=375, filesize=12.0 K 2024-12-05T07:40:16,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/60a7ccdf8a6c43e790c1ebe2c515d678 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/60a7ccdf8a6c43e790c1ebe2c515d678 2024-12-05T07:40:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T07:40:16,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/60a7ccdf8a6c43e790c1ebe2c515d678, entries=150, sequenceid=375, filesize=12.0 K 2024-12-05T07:40:16,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 27ae0e4955183fa53f189012e681dac5 in 595ms, sequenceid=375, compaction requested=false 2024-12-05T07:40:16,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:16,511 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:16,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-05T07:40:16,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/0edeaf08c2434a9098fba8b0f58f5d39 is 50, key is test_row_0/A:col10/1733384415879/Put/seqid=0 2024-12-05T07:40:16,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741917_1093 (size=12301) 2024-12-05T07:40:16,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384476553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384476554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384476556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T07:40:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:16,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384476559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384476559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384476665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384476666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384476666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384476666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384476676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T07:40:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:16,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T07:40:16,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T07:40:16,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:16,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:16,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:16,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:16,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384476876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384476876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384476876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384476876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384476885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:16,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/0edeaf08c2434a9098fba8b0f58f5d39 2024-12-05T07:40:16,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/be77be87b4db47dab2455b0936180b0e is 50, key is test_row_0/B:col10/1733384415879/Put/seqid=0 2024-12-05T07:40:16,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741918_1094 (size=12301) 2024-12-05T07:40:16,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/be77be87b4db47dab2455b0936180b0e 2024-12-05T07:40:16,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/f1924ae965c64b21ac9456304316ae4b is 50, key is test_row_0/C:col10/1733384415879/Put/seqid=0 2024-12-05T07:40:16,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741919_1095 (size=12301) 2024-12-05T07:40:17,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T07:40:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:17,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:17,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:17,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:17,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:17,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T07:40:17,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:17,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:17,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:17,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:17,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:17,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384477186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384477186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384477187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384477188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384477196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T07:40:17,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T07:40:17,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:17,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:17,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:17,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:17,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:17,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/f1924ae965c64b21ac9456304316ae4b 2024-12-05T07:40:17,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/0edeaf08c2434a9098fba8b0f58f5d39 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/0edeaf08c2434a9098fba8b0f58f5d39 2024-12-05T07:40:17,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/0edeaf08c2434a9098fba8b0f58f5d39, entries=150, sequenceid=402, filesize=12.0 K 2024-12-05T07:40:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/be77be87b4db47dab2455b0936180b0e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/be77be87b4db47dab2455b0936180b0e 2024-12-05T07:40:17,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/be77be87b4db47dab2455b0936180b0e, entries=150, sequenceid=402, filesize=12.0 K 2024-12-05T07:40:17,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/f1924ae965c64b21ac9456304316ae4b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/f1924ae965c64b21ac9456304316ae4b 2024-12-05T07:40:17,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/f1924ae965c64b21ac9456304316ae4b, entries=150, sequenceid=402, filesize=12.0 K 2024-12-05T07:40:17,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 27ae0e4955183fa53f189012e681dac5 in 900ms, sequenceid=402, compaction requested=true 2024-12-05T07:40:17,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:17,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:17,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:17,412 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:17,412 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:17,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:17,413 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:17,413 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/B is initiating minor compaction (all files) 2024-12-05T07:40:17,413 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/B in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:17,413 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c5fa6e49c1254bd4a3651ffcd2c2ab3c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/7f0b1e1f6d8844dbabeb6128b661214a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/be77be87b4db47dab2455b0936180b0e] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=36.9 K 2024-12-05T07:40:17,414 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:17,414 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/A is initiating minor compaction (all files) 2024-12-05T07:40:17,414 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/A in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:17,414 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a467398e5ac84553b873104f7d3024e3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/dab7854c0ad94a3982f09366abfafa3e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/0edeaf08c2434a9098fba8b0f58f5d39] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=41.6 K 2024-12-05T07:40:17,414 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting c5fa6e49c1254bd4a3651ffcd2c2ab3c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733384415051 2024-12-05T07:40:17,414 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a467398e5ac84553b873104f7d3024e3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733384415051 2024-12-05T07:40:17,414 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f0b1e1f6d8844dbabeb6128b661214a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733384415190 2024-12-05T07:40:17,415 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting dab7854c0ad94a3982f09366abfafa3e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733384415185 2024-12-05T07:40:17,415 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting be77be87b4db47dab2455b0936180b0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733384415868 2024-12-05T07:40:17,415 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0edeaf08c2434a9098fba8b0f58f5d39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733384415868 2024-12-05T07:40:17,428 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#A#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:17,429 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/9886197df47a44b3932b9277fb111f4d is 50, key is test_row_0/A:col10/1733384415879/Put/seqid=0 2024-12-05T07:40:17,432 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:17,433 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/0069d0d2b08e4e2f8a9a63f82200e897 is 50, key is test_row_0/B:col10/1733384415879/Put/seqid=0 2024-12-05T07:40:17,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741920_1096 (size=13255) 2024-12-05T07:40:17,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741921_1097 (size=13255) 2024-12-05T07:40:17,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T07:40:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:17,483 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T07:40:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/141d9e7bd4d5421b8f44f3434dd4dab5 is 50, key is test_row_0/A:col10/1733384416556/Put/seqid=0 2024-12-05T07:40:17,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741922_1098 (size=12301) 2024-12-05T07:40:17,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:17,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:17,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384477737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384477739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384477740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384477745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384477747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,849 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/9886197df47a44b3932b9277fb111f4d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/9886197df47a44b3932b9277fb111f4d 2024-12-05T07:40:17,859 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/0069d0d2b08e4e2f8a9a63f82200e897 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/0069d0d2b08e4e2f8a9a63f82200e897 2024-12-05T07:40:17,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384477857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384477857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384477857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384477858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:17,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384477858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:17,869 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/A of 27ae0e4955183fa53f189012e681dac5 into 9886197df47a44b3932b9277fb111f4d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:17,869 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:17,869 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/A, priority=13, startTime=1733384417412; duration=0sec 2024-12-05T07:40:17,869 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:17,869 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:17,870 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:17,872 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/B of 27ae0e4955183fa53f189012e681dac5 into 0069d0d2b08e4e2f8a9a63f82200e897(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:17,872 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:17,872 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/B, priority=13, startTime=1733384417412; duration=0sec 2024-12-05T07:40:17,873 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:17,873 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:17,874 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:17,874 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/C is initiating minor compaction (all files) 2024-12-05T07:40:17,874 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/C in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:17,874 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/3c7e7f8ea34e40b8b8cc0aa623c3fcb1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/60a7ccdf8a6c43e790c1ebe2c515d678, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/f1924ae965c64b21ac9456304316ae4b] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=36.8 K 2024-12-05T07:40:17,875 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c7e7f8ea34e40b8b8cc0aa623c3fcb1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733384415051 2024-12-05T07:40:17,875 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60a7ccdf8a6c43e790c1ebe2c515d678, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1733384415190 2024-12-05T07:40:17,876 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1924ae965c64b21ac9456304316ae4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733384415868 2024-12-05T07:40:17,893 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:17,894 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/169729decfec4a2b97f97fb0bb66005b is 50, key is test_row_0/C:col10/1733384415879/Put/seqid=0 2024-12-05T07:40:17,907 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/141d9e7bd4d5421b8f44f3434dd4dab5 2024-12-05T07:40:17,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/c3957d99f3a34b5a94d1273140ae2d66 is 50, key is test_row_0/B:col10/1733384416556/Put/seqid=0 2024-12-05T07:40:17,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741923_1099 (size=13221) 2024-12-05T07:40:17,961 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/169729decfec4a2b97f97fb0bb66005b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/169729decfec4a2b97f97fb0bb66005b 2024-12-05T07:40:17,969 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/C of 27ae0e4955183fa53f189012e681dac5 into 169729decfec4a2b97f97fb0bb66005b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:17,969 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:17,970 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/C, priority=13, startTime=1733384417413; duration=0sec 2024-12-05T07:40:17,970 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:17,970 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:C 2024-12-05T07:40:17,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741924_1100 (size=12301) 2024-12-05T07:40:17,979 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/c3957d99f3a34b5a94d1273140ae2d66 2024-12-05T07:40:18,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/a9701d63ca4c4a8fa09a6c10ba3e3f63 is 50, key is test_row_0/C:col10/1733384416556/Put/seqid=0 2024-12-05T07:40:18,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741925_1101 (size=12301) 2024-12-05T07:40:18,061 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/a9701d63ca4c4a8fa09a6c10ba3e3f63 2024-12-05T07:40:18,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/141d9e7bd4d5421b8f44f3434dd4dab5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/141d9e7bd4d5421b8f44f3434dd4dab5 2024-12-05T07:40:18,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384478076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384478077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384478077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384478077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384478077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,080 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/141d9e7bd4d5421b8f44f3434dd4dab5, entries=150, sequenceid=415, filesize=12.0 K 2024-12-05T07:40:18,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/c3957d99f3a34b5a94d1273140ae2d66 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c3957d99f3a34b5a94d1273140ae2d66 2024-12-05T07:40:18,094 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c3957d99f3a34b5a94d1273140ae2d66, entries=150, sequenceid=415, filesize=12.0 K 2024-12-05T07:40:18,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/a9701d63ca4c4a8fa09a6c10ba3e3f63 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a9701d63ca4c4a8fa09a6c10ba3e3f63 2024-12-05T07:40:18,107 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a9701d63ca4c4a8fa09a6c10ba3e3f63, entries=150, sequenceid=415, filesize=12.0 K 2024-12-05T07:40:18,112 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 27ae0e4955183fa53f189012e681dac5 in 629ms, sequenceid=415, compaction requested=false 2024-12-05T07:40:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:18,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-05T07:40:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-05T07:40:18,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-05T07:40:18,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0170 sec 2024-12-05T07:40:18,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.0280 sec 2024-12-05T07:40:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T07:40:18,246 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:18,246 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-05T07:40:18,248 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:18,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T07:40:18,249 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:18,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T07:40:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:18,388 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:18,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T07:40:18,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/04241fe6ad8b41b49689e3dc29913d92 is 50, key is test_row_0/A:col10/1733384418387/Put/seqid=0 2024-12-05T07:40:18,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T07:40:18,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:18,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:18,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:18,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384478394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384478396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384478396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384478401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384478400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741926_1102 (size=14741) 2024-12-05T07:40:18,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384478506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384478516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384478516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384478516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384478516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T07:40:18,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:18,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:18,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:18,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T07:40:18,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T07:40:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:18,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384478716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384478725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384478726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384478726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384478726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:18,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/04241fe6ad8b41b49689e3dc29913d92 2024-12-05T07:40:18,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/a51e2acf25654870975493d3efea7eec is 50, key is test_row_0/B:col10/1733384418387/Put/seqid=0 2024-12-05T07:40:18,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741927_1103 (size=12301) 2024-12-05T07:40:18,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T07:40:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:18,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T07:40:19,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384479026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:19,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T07:40:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384479035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:19,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384479036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:19,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:19,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384479036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:19,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:19,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384479036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:19,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:19,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T07:40:19,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:19,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/a51e2acf25654870975493d3efea7eec 2024-12-05T07:40:19,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/8549d45587c845aa8259d71498a6ec9a is 50, key is test_row_0/C:col10/1733384418387/Put/seqid=0 2024-12-05T07:40:19,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741928_1104 (size=12301) 2024-12-05T07:40:19,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T07:40:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T07:40:19,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T07:40:19,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:19,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59762 deadline: 1733384479536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:19,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59796 deadline: 1733384479546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59806 deadline: 1733384479546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59818 deadline: 1733384479546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59760 deadline: 1733384479547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:19,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T07:40:19,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. as already flushing 2024-12-05T07:40:19,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:19,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/8549d45587c845aa8259d71498a6ec9a 2024-12-05T07:40:19,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/04241fe6ad8b41b49689e3dc29913d92 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/04241fe6ad8b41b49689e3dc29913d92 2024-12-05T07:40:19,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/04241fe6ad8b41b49689e3dc29913d92, entries=200, sequenceid=443, filesize=14.4 K 2024-12-05T07:40:19,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/a51e2acf25654870975493d3efea7eec as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/a51e2acf25654870975493d3efea7eec 2024-12-05T07:40:19,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/a51e2acf25654870975493d3efea7eec, entries=150, sequenceid=443, filesize=12.0 K 2024-12-05T07:40:19,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/8549d45587c845aa8259d71498a6ec9a as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8549d45587c845aa8259d71498a6ec9a 2024-12-05T07:40:19,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8549d45587c845aa8259d71498a6ec9a, entries=150, sequenceid=443, filesize=12.0 K 2024-12-05T07:40:19,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 27ae0e4955183fa53f189012e681dac5 in 1295ms, sequenceid=443, compaction requested=true 2024-12-05T07:40:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:19,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:19,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:19,685 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:19,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:19,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:19,685 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:19,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 27ae0e4955183fa53f189012e681dac5:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:19,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:19,686 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:19,686 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:19,686 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/B is initiating minor compaction (all files) 2024-12-05T07:40:19,686 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/A is initiating minor compaction (all files) 2024-12-05T07:40:19,686 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/B in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,686 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/A in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,686 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/9886197df47a44b3932b9277fb111f4d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/141d9e7bd4d5421b8f44f3434dd4dab5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/04241fe6ad8b41b49689e3dc29913d92] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=39.4 K 2024-12-05T07:40:19,686 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/0069d0d2b08e4e2f8a9a63f82200e897, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c3957d99f3a34b5a94d1273140ae2d66, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/a51e2acf25654870975493d3efea7eec] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=37.0 K 2024-12-05T07:40:19,687 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9886197df47a44b3932b9277fb111f4d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733384415868 2024-12-05T07:40:19,687 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0069d0d2b08e4e2f8a9a63f82200e897, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733384415868 2024-12-05T07:40:19,687 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 141d9e7bd4d5421b8f44f3434dd4dab5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733384416519 2024-12-05T07:40:19,687 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting c3957d99f3a34b5a94d1273140ae2d66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733384416519 2024-12-05T07:40:19,687 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a51e2acf25654870975493d3efea7eec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733384417734 2024-12-05T07:40:19,687 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04241fe6ad8b41b49689e3dc29913d92, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733384417734 2024-12-05T07:40:19,700 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#A#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:19,700 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:19,701 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/b5c0278f43704344bdf07943a0d5b7e5 is 50, key is test_row_0/B:col10/1733384418387/Put/seqid=0 2024-12-05T07:40:19,701 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/2be408ef847a423b973272f338ca186f is 50, key is test_row_0/A:col10/1733384418387/Put/seqid=0 2024-12-05T07:40:19,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741930_1106 (size=13357) 2024-12-05T07:40:19,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741929_1105 (size=13357) 2024-12-05T07:40:19,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-05T07:40:19,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,794 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:40:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/2fdc6192cbd0449ea8307b4663d08ab5 is 50, key is test_row_0/A:col10/1733384418394/Put/seqid=0 2024-12-05T07:40:19,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741931_1107 (size=12301) 2024-12-05T07:40:19,880 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/2fdc6192cbd0449ea8307b4663d08ab5 2024-12-05T07:40:19,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/73db5e7d0c2e4e17b65790b9245a3800 is 50, key is test_row_0/B:col10/1733384418394/Put/seqid=0 2024-12-05T07:40:19,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741932_1108 (size=12301) 2024-12-05T07:40:19,904 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/73db5e7d0c2e4e17b65790b9245a3800 2024-12-05T07:40:19,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/a5334c85f71b443da875b401d3b8102f is 50, key is test_row_0/C:col10/1733384418394/Put/seqid=0 2024-12-05T07:40:19,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741933_1109 (size=12301) 2024-12-05T07:40:19,925 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/a5334c85f71b443da875b401d3b8102f 2024-12-05T07:40:19,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/2fdc6192cbd0449ea8307b4663d08ab5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/2fdc6192cbd0449ea8307b4663d08ab5 2024-12-05T07:40:19,940 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/2fdc6192cbd0449ea8307b4663d08ab5, entries=150, sequenceid=454, filesize=12.0 K 2024-12-05T07:40:19,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/73db5e7d0c2e4e17b65790b9245a3800 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/73db5e7d0c2e4e17b65790b9245a3800 2024-12-05T07:40:19,950 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/73db5e7d0c2e4e17b65790b9245a3800, entries=150, sequenceid=454, filesize=12.0 K 2024-12-05T07:40:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/a5334c85f71b443da875b401d3b8102f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a5334c85f71b443da875b401d3b8102f 2024-12-05T07:40:19,958 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a5334c85f71b443da875b401d3b8102f, entries=150, sequenceid=454, filesize=12.0 K 2024-12-05T07:40:19,959 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 27ae0e4955183fa53f189012e681dac5 in 165ms, sequenceid=454, compaction requested=true 2024-12-05T07:40:19,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:19,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:19,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-05T07:40:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-05T07:40:19,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-05T07:40:19,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7120 sec 2024-12-05T07:40:19,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.7180 sec 2024-12-05T07:40:20,131 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/2be408ef847a423b973272f338ca186f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/2be408ef847a423b973272f338ca186f 2024-12-05T07:40:20,133 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/b5c0278f43704344bdf07943a0d5b7e5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b5c0278f43704344bdf07943a0d5b7e5 2024-12-05T07:40:20,147 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/B of 27ae0e4955183fa53f189012e681dac5 into b5c0278f43704344bdf07943a0d5b7e5(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:20,147 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:20,147 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/B, priority=13, startTime=1733384419685; duration=0sec 2024-12-05T07:40:20,147 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:20,147 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/A of 27ae0e4955183fa53f189012e681dac5 into 2be408ef847a423b973272f338ca186f(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:20,147 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:B 2024-12-05T07:40:20,147 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:20,147 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/A, priority=13, startTime=1733384419684; duration=0sec 2024-12-05T07:40:20,147 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:20,147 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:20,148 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:A 2024-12-05T07:40:20,149 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:20,149 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 27ae0e4955183fa53f189012e681dac5/C is initiating minor compaction (all files) 2024-12-05T07:40:20,149 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 27ae0e4955183fa53f189012e681dac5/C in TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:20,149 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/169729decfec4a2b97f97fb0bb66005b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a9701d63ca4c4a8fa09a6c10ba3e3f63, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8549d45587c845aa8259d71498a6ec9a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a5334c85f71b443da875b401d3b8102f] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp, totalSize=48.9 K 2024-12-05T07:40:20,150 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 169729decfec4a2b97f97fb0bb66005b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733384415868 2024-12-05T07:40:20,151 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a9701d63ca4c4a8fa09a6c10ba3e3f63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1733384416519 2024-12-05T07:40:20,151 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8549d45587c845aa8259d71498a6ec9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1733384417734 2024-12-05T07:40:20,152 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a5334c85f71b443da875b401d3b8102f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1733384418394 2024-12-05T07:40:20,187 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 27ae0e4955183fa53f189012e681dac5#C#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:20,191 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/03c1d6d01e544eeb9782166e928664b3 is 50, key is test_row_0/C:col10/1733384418394/Put/seqid=0 2024-12-05T07:40:20,196 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-05T07:40:20,196 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:20,199 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-05T07:40:20,199 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:20,200 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-05T07:40:20,200 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:20,202 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,202 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,202 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,202 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,203 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,203 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:20,203 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:20,204 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:20,208 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-05T07:40:20,208 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:20,208 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,208 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741934_1110 (size=13357) 2024-12-05T07:40:20,228 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/03c1d6d01e544eeb9782166e928664b3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/03c1d6d01e544eeb9782166e928664b3 2024-12-05T07:40:20,239 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 27ae0e4955183fa53f189012e681dac5/C of 27ae0e4955183fa53f189012e681dac5 into 03c1d6d01e544eeb9782166e928664b3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:20,239 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 27ae0e4955183fa53f189012e681dac5: 2024-12-05T07:40:20,239 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., storeName=27ae0e4955183fa53f189012e681dac5/C, priority=12, startTime=1733384419685; duration=0sec 2024-12-05T07:40:20,239 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:20,239 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 27ae0e4955183fa53f189012e681dac5:C 2024-12-05T07:40:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T07:40:20,396 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:20,596 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-05T07:40:20,596 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-05T07:40:20,596 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-05T07:40:20,596 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:20,596 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:20,596 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:20,596 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,596 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,596 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,596 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,596 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,596 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:20,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:20,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:20,598 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-05T07:40:20,598 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:20,598 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,598 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:20,598 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-05T07:40:20,598 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:20,598 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,599 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:20,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T07:40:20,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-05T07:40:20,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-12-05T07:40:20,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-12-05T07:40:20,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-12-05T07:40:20,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-05T07:40:20,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T07:40:20,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3411 2024-12-05T07:40:20,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3380 2024-12-05T07:40:20,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T07:40:20,599 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1637 2024-12-05T07:40:20,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4902 rows 2024-12-05T07:40:20,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1633 2024-12-05T07:40:20,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4898 rows 2024-12-05T07:40:20,600 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T07:40:20,600 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T07:40:20,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:20,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:20,605 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T07:40:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:20,613 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384420613"}]},"ts":"1733384420613"} 2024-12-05T07:40:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T07:40:20,615 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T07:40:20,615 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T07:40:20,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T07:40:20,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=27ae0e4955183fa53f189012e681dac5, UNASSIGN}] 2024-12-05T07:40:20,624 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=27ae0e4955183fa53f189012e681dac5, UNASSIGN 2024-12-05T07:40:20,625 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33643 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fc6cd38557f3,36693,1733384394401, table=TestAcidGuarantees, region=27ae0e4955183fa53f189012e681dac5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T07:40:20,626 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=27ae0e4955183fa53f189012e681dac5, regionState=CLOSING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:20,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=27ae0e4955183fa53f189012e681dac5, UNASSIGN because future has completed 2024-12-05T07:40:20,629 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T07:40:20,629 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:40:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T07:40:20,787 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:20,787 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T07:40:20,788 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 27ae0e4955183fa53f189012e681dac5, disabling compactions & flushes 2024-12-05T07:40:20,788 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:20,788 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:20,788 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. after waiting 0 ms 2024-12-05T07:40:20,788 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:20,788 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 27ae0e4955183fa53f189012e681dac5 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-05T07:40:20,795 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/77a0d84757bb47cb9a9a5ab65b62aeff is 50, key is test_row_0/A:col10/1733384420596/Put/seqid=0 2024-12-05T07:40:20,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741935_1111 (size=12301) 2024-12-05T07:40:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T07:40:21,202 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/77a0d84757bb47cb9a9a5ab65b62aeff 2024-12-05T07:40:21,215 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/426e9adfc41247de88a0dc61f87270a5 is 50, key is test_row_0/B:col10/1733384420596/Put/seqid=0 2024-12-05T07:40:21,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741936_1112 (size=12301) 2024-12-05T07:40:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T07:40:21,621 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/426e9adfc41247de88a0dc61f87270a5 2024-12-05T07:40:21,632 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/36e8b92a498d44d087c459ac761f6870 is 50, key is test_row_0/C:col10/1733384420596/Put/seqid=0 2024-12-05T07:40:21,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741937_1113 (size=12301) 2024-12-05T07:40:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T07:40:22,036 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/36e8b92a498d44d087c459ac761f6870 2024-12-05T07:40:22,041 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/A/77a0d84757bb47cb9a9a5ab65b62aeff as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/77a0d84757bb47cb9a9a5ab65b62aeff 2024-12-05T07:40:22,046 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/77a0d84757bb47cb9a9a5ab65b62aeff, entries=150, sequenceid=465, filesize=12.0 K 2024-12-05T07:40:22,047 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/B/426e9adfc41247de88a0dc61f87270a5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/426e9adfc41247de88a0dc61f87270a5 2024-12-05T07:40:22,053 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/426e9adfc41247de88a0dc61f87270a5, entries=150, sequenceid=465, filesize=12.0 K 2024-12-05T07:40:22,054 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/.tmp/C/36e8b92a498d44d087c459ac761f6870 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/36e8b92a498d44d087c459ac761f6870 2024-12-05T07:40:22,058 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/36e8b92a498d44d087c459ac761f6870, entries=150, sequenceid=465, filesize=12.0 K 2024-12-05T07:40:22,059 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 27ae0e4955183fa53f189012e681dac5 in 1271ms, sequenceid=465, compaction requested=true 2024-12-05T07:40:22,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/1185a997cd6d400da34d8fe64deddaf3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d08c84990b1a4e57970ec51b355707db, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/9c3d497eefa241ffbdbc9433883c59e6, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/68413359cebd4798bf0a91eb236d3d92, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d8a33272a2ac4365ac016ebdf6a56696, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/118a2f5d1c0349a3895f221f9c65672f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/240c30feed4a40fb994baad76191adba, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a82e7fe6410942888a6782564442d564, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/e00c8623f92c4b8c89da982592607296, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/3dbef7207fc54b0a9d7b3469094867e0, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f2cba269afb6404eafcd8d09dbd932ea, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/420fa540985a4f5cbff22ca64008c18a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/ee7042d3e0e24fdfa3374cb2dd739943, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/aeeb151e2f944b4dad6b05f0d582d3fb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/93f9bb8192d54e7286c00e4bbd68a537, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/433aaf63ab794a2fac61fd14816bc602, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/63b8f3b303c146159855927e1669fa4e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/06159bdd3b62434fb87041620543c254, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/75935a6dacae40ccbc0151de3aff62bd, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/abacc1b2dfff47e3828a7ddf26a90814, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/81ebc80656594cf081200912aa1ca419, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/7573284aa2e54b7aad113ab87e6f301e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/06a18e99dca74871bd563385589ee16f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a7ee7d768b5542b6b326ce87ace12164, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f5de23a7da0e4893aab3937aba707ee4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a467398e5ac84553b873104f7d3024e3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/dab7854c0ad94a3982f09366abfafa3e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/9886197df47a44b3932b9277fb111f4d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/0edeaf08c2434a9098fba8b0f58f5d39, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/141d9e7bd4d5421b8f44f3434dd4dab5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/04241fe6ad8b41b49689e3dc29913d92] to archive 2024-12-05T07:40:22,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:40:22,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/1185a997cd6d400da34d8fe64deddaf3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/1185a997cd6d400da34d8fe64deddaf3 2024-12-05T07:40:22,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d08c84990b1a4e57970ec51b355707db to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d08c84990b1a4e57970ec51b355707db 2024-12-05T07:40:22,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/9c3d497eefa241ffbdbc9433883c59e6 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/9c3d497eefa241ffbdbc9433883c59e6 2024-12-05T07:40:22,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/68413359cebd4798bf0a91eb236d3d92 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/68413359cebd4798bf0a91eb236d3d92 2024-12-05T07:40:22,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d8a33272a2ac4365ac016ebdf6a56696 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/d8a33272a2ac4365ac016ebdf6a56696 2024-12-05T07:40:22,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/118a2f5d1c0349a3895f221f9c65672f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/118a2f5d1c0349a3895f221f9c65672f 2024-12-05T07:40:22,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/240c30feed4a40fb994baad76191adba to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/240c30feed4a40fb994baad76191adba 2024-12-05T07:40:22,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a82e7fe6410942888a6782564442d564 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a82e7fe6410942888a6782564442d564 2024-12-05T07:40:22,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/e00c8623f92c4b8c89da982592607296 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/e00c8623f92c4b8c89da982592607296 2024-12-05T07:40:22,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/3dbef7207fc54b0a9d7b3469094867e0 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/3dbef7207fc54b0a9d7b3469094867e0 2024-12-05T07:40:22,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f2cba269afb6404eafcd8d09dbd932ea to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f2cba269afb6404eafcd8d09dbd932ea 2024-12-05T07:40:22,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/420fa540985a4f5cbff22ca64008c18a to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/420fa540985a4f5cbff22ca64008c18a 2024-12-05T07:40:22,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/ee7042d3e0e24fdfa3374cb2dd739943 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/ee7042d3e0e24fdfa3374cb2dd739943 2024-12-05T07:40:22,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/aeeb151e2f944b4dad6b05f0d582d3fb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/aeeb151e2f944b4dad6b05f0d582d3fb 2024-12-05T07:40:22,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/93f9bb8192d54e7286c00e4bbd68a537 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/93f9bb8192d54e7286c00e4bbd68a537 2024-12-05T07:40:22,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/433aaf63ab794a2fac61fd14816bc602 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/433aaf63ab794a2fac61fd14816bc602 2024-12-05T07:40:22,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/63b8f3b303c146159855927e1669fa4e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/63b8f3b303c146159855927e1669fa4e 2024-12-05T07:40:22,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/06159bdd3b62434fb87041620543c254 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/06159bdd3b62434fb87041620543c254 2024-12-05T07:40:22,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/75935a6dacae40ccbc0151de3aff62bd to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/75935a6dacae40ccbc0151de3aff62bd 2024-12-05T07:40:22,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/abacc1b2dfff47e3828a7ddf26a90814 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/abacc1b2dfff47e3828a7ddf26a90814 2024-12-05T07:40:22,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/81ebc80656594cf081200912aa1ca419 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/81ebc80656594cf081200912aa1ca419 2024-12-05T07:40:22,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/7573284aa2e54b7aad113ab87e6f301e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/7573284aa2e54b7aad113ab87e6f301e 2024-12-05T07:40:22,115 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/06a18e99dca74871bd563385589ee16f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/06a18e99dca74871bd563385589ee16f 2024-12-05T07:40:22,117 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a7ee7d768b5542b6b326ce87ace12164 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a7ee7d768b5542b6b326ce87ace12164 2024-12-05T07:40:22,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f5de23a7da0e4893aab3937aba707ee4 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/f5de23a7da0e4893aab3937aba707ee4 2024-12-05T07:40:22,121 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a467398e5ac84553b873104f7d3024e3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/a467398e5ac84553b873104f7d3024e3 2024-12-05T07:40:22,123 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/dab7854c0ad94a3982f09366abfafa3e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/dab7854c0ad94a3982f09366abfafa3e 2024-12-05T07:40:22,125 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/9886197df47a44b3932b9277fb111f4d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/9886197df47a44b3932b9277fb111f4d 2024-12-05T07:40:22,126 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/0edeaf08c2434a9098fba8b0f58f5d39 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/0edeaf08c2434a9098fba8b0f58f5d39 2024-12-05T07:40:22,128 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/141d9e7bd4d5421b8f44f3434dd4dab5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/141d9e7bd4d5421b8f44f3434dd4dab5 2024-12-05T07:40:22,130 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/04241fe6ad8b41b49689e3dc29913d92 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/04241fe6ad8b41b49689e3dc29913d92 2024-12-05T07:40:22,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/f00bb68559e34bd3a8e839c34b31b9ac, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9af2c5363c3c4aef9223ebcef11dc8ac, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/079b3d1276c04336a0f773c12e8644bb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9538e65d9bd94aa6a9723982a6b79b71, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/4e39f7b2c862460cbcc16996ddcc256c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/1f22c6eec3844e91aad875cfcf92e2fb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/6a54f43c3ae1432088b5fe1f408d2afe, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/753072476fa4424e881a5fd8c5393594, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/41b89a4daa9f49dc8d604d49cf65173d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/64cbef27535643b585b0b3f8b02c6573, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/2a2300d27848466086c569b34a660089, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/a93edb60351844d3be06f414f88c75bf, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/081621c4f8fd4a5d83a0d75f151ba440, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/70baa525844b43569a1774a1af65e581, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/cf0735684e2e428e919993917a4a278f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/ce5380b2c1d24d0199456a2bc94dd509, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b9836cadbc4e400da4793ab46a3d75b7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/08ca77aa0ced4bb1a2c58a552b20ac83, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/d4ef570606234b8ab210cb9196c40d81, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/5ce4ef2f22774cbcad2fde7e6c9c45f6, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/767449440e4346ca9b93c7c7fa121f84, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/ff0b562f8c024d3898b6288d365b154e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/06e9931838ae47dab57d7153943ce9d7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b461021ab40c42fda69b2743e1b516bb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c5fa6e49c1254bd4a3651ffcd2c2ab3c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c439b7de64c44a0cb547103fd071820f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/7f0b1e1f6d8844dbabeb6128b661214a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/0069d0d2b08e4e2f8a9a63f82200e897, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/be77be87b4db47dab2455b0936180b0e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c3957d99f3a34b5a94d1273140ae2d66, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/a51e2acf25654870975493d3efea7eec] to archive 2024-12-05T07:40:22,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:40:22,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/f00bb68559e34bd3a8e839c34b31b9ac to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/f00bb68559e34bd3a8e839c34b31b9ac 2024-12-05T07:40:22,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9af2c5363c3c4aef9223ebcef11dc8ac to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9af2c5363c3c4aef9223ebcef11dc8ac 2024-12-05T07:40:22,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/079b3d1276c04336a0f773c12e8644bb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/079b3d1276c04336a0f773c12e8644bb 2024-12-05T07:40:22,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9538e65d9bd94aa6a9723982a6b79b71 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/9538e65d9bd94aa6a9723982a6b79b71 2024-12-05T07:40:22,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/4e39f7b2c862460cbcc16996ddcc256c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/4e39f7b2c862460cbcc16996ddcc256c 2024-12-05T07:40:22,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/1f22c6eec3844e91aad875cfcf92e2fb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/1f22c6eec3844e91aad875cfcf92e2fb 2024-12-05T07:40:22,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/6a54f43c3ae1432088b5fe1f408d2afe to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/6a54f43c3ae1432088b5fe1f408d2afe 2024-12-05T07:40:22,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/753072476fa4424e881a5fd8c5393594 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/753072476fa4424e881a5fd8c5393594 2024-12-05T07:40:22,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/41b89a4daa9f49dc8d604d49cf65173d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/41b89a4daa9f49dc8d604d49cf65173d 2024-12-05T07:40:22,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/64cbef27535643b585b0b3f8b02c6573 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/64cbef27535643b585b0b3f8b02c6573 2024-12-05T07:40:22,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/2a2300d27848466086c569b34a660089 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/2a2300d27848466086c569b34a660089 2024-12-05T07:40:22,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/a93edb60351844d3be06f414f88c75bf to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/a93edb60351844d3be06f414f88c75bf 2024-12-05T07:40:22,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/081621c4f8fd4a5d83a0d75f151ba440 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/081621c4f8fd4a5d83a0d75f151ba440 2024-12-05T07:40:22,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/70baa525844b43569a1774a1af65e581 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/70baa525844b43569a1774a1af65e581 2024-12-05T07:40:22,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/cf0735684e2e428e919993917a4a278f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/cf0735684e2e428e919993917a4a278f 2024-12-05T07:40:22,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/ce5380b2c1d24d0199456a2bc94dd509 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/ce5380b2c1d24d0199456a2bc94dd509 2024-12-05T07:40:22,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b9836cadbc4e400da4793ab46a3d75b7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b9836cadbc4e400da4793ab46a3d75b7 2024-12-05T07:40:22,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/08ca77aa0ced4bb1a2c58a552b20ac83 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/08ca77aa0ced4bb1a2c58a552b20ac83 2024-12-05T07:40:22,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/d4ef570606234b8ab210cb9196c40d81 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/d4ef570606234b8ab210cb9196c40d81 2024-12-05T07:40:22,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/5ce4ef2f22774cbcad2fde7e6c9c45f6 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/5ce4ef2f22774cbcad2fde7e6c9c45f6 2024-12-05T07:40:22,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/767449440e4346ca9b93c7c7fa121f84 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/767449440e4346ca9b93c7c7fa121f84 2024-12-05T07:40:22,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/ff0b562f8c024d3898b6288d365b154e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/ff0b562f8c024d3898b6288d365b154e 2024-12-05T07:40:22,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/06e9931838ae47dab57d7153943ce9d7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/06e9931838ae47dab57d7153943ce9d7 2024-12-05T07:40:22,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b461021ab40c42fda69b2743e1b516bb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b461021ab40c42fda69b2743e1b516bb 2024-12-05T07:40:22,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c5fa6e49c1254bd4a3651ffcd2c2ab3c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c5fa6e49c1254bd4a3651ffcd2c2ab3c 2024-12-05T07:40:22,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c439b7de64c44a0cb547103fd071820f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c439b7de64c44a0cb547103fd071820f 2024-12-05T07:40:22,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/7f0b1e1f6d8844dbabeb6128b661214a to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/7f0b1e1f6d8844dbabeb6128b661214a 2024-12-05T07:40:22,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/0069d0d2b08e4e2f8a9a63f82200e897 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/0069d0d2b08e4e2f8a9a63f82200e897 2024-12-05T07:40:22,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/be77be87b4db47dab2455b0936180b0e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/be77be87b4db47dab2455b0936180b0e 2024-12-05T07:40:22,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c3957d99f3a34b5a94d1273140ae2d66 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/c3957d99f3a34b5a94d1273140ae2d66 2024-12-05T07:40:22,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/a51e2acf25654870975493d3efea7eec to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/a51e2acf25654870975493d3efea7eec 2024-12-05T07:40:22,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c0f39cfda9b54613a2096328363f9251, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/0db9fcc7c63544cdbebf5d8bf9786787, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/5c00ecc705504deb822d260a17898e11, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/58ce132f8d92435e8f039830cd0bfd4e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/43716f4175eb4306a5e14ad20ce60a46, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/52777ccf3c1847d6a85a12c184b27d2c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ce65850ebcef4f25b248c935040db891, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/4e09bd4997264eeeba67c7060c165199, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/80453c9a701c4def9b1f69833bb708b8, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/1c5ed17aa5c54e41a213742481a4fdbd, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/eadff0b33a4a48d28b4f6c929b24754e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/08111a377326478d85678865f6ef44dd, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ccd846ec4f3e4f85a64509e820da93b4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a8cf4fad05644eb682de19e6db6a557e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8dca70746f2f4a73bcaf0a7d891de2b2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/3c44b8f4d3204892908e7d4a8ec53545, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/21d0be3fd4194585ae2f82b26902b7b2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/4764f65c0af04455989f441a56465370, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a512d5e071af422a83d238d31e5c5f29, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/e2c5fd4e5c464cc8ae863f057066ab78, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/2fad64e71407482596820dc121f75950, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c68c7acd46024fe39620067460e84fa6, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8f2cf6ef951e4c05959f2e0bde915c90, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/3c7e7f8ea34e40b8b8cc0aa623c3fcb1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/9cb6916699474e35be6a652e5345323a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/60a7ccdf8a6c43e790c1ebe2c515d678, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/169729decfec4a2b97f97fb0bb66005b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/f1924ae965c64b21ac9456304316ae4b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a9701d63ca4c4a8fa09a6c10ba3e3f63, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8549d45587c845aa8259d71498a6ec9a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a5334c85f71b443da875b401d3b8102f] to archive 2024-12-05T07:40:22,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:40:22,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c0f39cfda9b54613a2096328363f9251 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c0f39cfda9b54613a2096328363f9251 2024-12-05T07:40:22,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/0db9fcc7c63544cdbebf5d8bf9786787 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/0db9fcc7c63544cdbebf5d8bf9786787 2024-12-05T07:40:22,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/5c00ecc705504deb822d260a17898e11 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/5c00ecc705504deb822d260a17898e11 2024-12-05T07:40:22,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/58ce132f8d92435e8f039830cd0bfd4e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/58ce132f8d92435e8f039830cd0bfd4e 2024-12-05T07:40:22,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/43716f4175eb4306a5e14ad20ce60a46 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/43716f4175eb4306a5e14ad20ce60a46 2024-12-05T07:40:22,202 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/52777ccf3c1847d6a85a12c184b27d2c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/52777ccf3c1847d6a85a12c184b27d2c 2024-12-05T07:40:22,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ce65850ebcef4f25b248c935040db891 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ce65850ebcef4f25b248c935040db891 2024-12-05T07:40:22,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/4e09bd4997264eeeba67c7060c165199 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/4e09bd4997264eeeba67c7060c165199 2024-12-05T07:40:22,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/80453c9a701c4def9b1f69833bb708b8 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/80453c9a701c4def9b1f69833bb708b8 2024-12-05T07:40:22,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/1c5ed17aa5c54e41a213742481a4fdbd to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/1c5ed17aa5c54e41a213742481a4fdbd 2024-12-05T07:40:22,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/eadff0b33a4a48d28b4f6c929b24754e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/eadff0b33a4a48d28b4f6c929b24754e 2024-12-05T07:40:22,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/08111a377326478d85678865f6ef44dd to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/08111a377326478d85678865f6ef44dd 2024-12-05T07:40:22,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ccd846ec4f3e4f85a64509e820da93b4 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/ccd846ec4f3e4f85a64509e820da93b4 2024-12-05T07:40:22,213 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a8cf4fad05644eb682de19e6db6a557e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a8cf4fad05644eb682de19e6db6a557e 2024-12-05T07:40:22,216 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8dca70746f2f4a73bcaf0a7d891de2b2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8dca70746f2f4a73bcaf0a7d891de2b2 2024-12-05T07:40:22,218 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/3c44b8f4d3204892908e7d4a8ec53545 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/3c44b8f4d3204892908e7d4a8ec53545 2024-12-05T07:40:22,220 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/21d0be3fd4194585ae2f82b26902b7b2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/21d0be3fd4194585ae2f82b26902b7b2 2024-12-05T07:40:22,222 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/4764f65c0af04455989f441a56465370 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/4764f65c0af04455989f441a56465370 2024-12-05T07:40:22,223 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a512d5e071af422a83d238d31e5c5f29 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a512d5e071af422a83d238d31e5c5f29 2024-12-05T07:40:22,224 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/e2c5fd4e5c464cc8ae863f057066ab78 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/e2c5fd4e5c464cc8ae863f057066ab78 2024-12-05T07:40:22,226 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/2fad64e71407482596820dc121f75950 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/2fad64e71407482596820dc121f75950 2024-12-05T07:40:22,228 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c68c7acd46024fe39620067460e84fa6 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/c68c7acd46024fe39620067460e84fa6 2024-12-05T07:40:22,230 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8f2cf6ef951e4c05959f2e0bde915c90 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8f2cf6ef951e4c05959f2e0bde915c90 2024-12-05T07:40:22,231 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/3c7e7f8ea34e40b8b8cc0aa623c3fcb1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/3c7e7f8ea34e40b8b8cc0aa623c3fcb1 2024-12-05T07:40:22,233 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/9cb6916699474e35be6a652e5345323a to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/9cb6916699474e35be6a652e5345323a 2024-12-05T07:40:22,234 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/60a7ccdf8a6c43e790c1ebe2c515d678 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/60a7ccdf8a6c43e790c1ebe2c515d678 2024-12-05T07:40:22,235 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/169729decfec4a2b97f97fb0bb66005b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/169729decfec4a2b97f97fb0bb66005b 2024-12-05T07:40:22,239 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/f1924ae965c64b21ac9456304316ae4b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/f1924ae965c64b21ac9456304316ae4b 2024-12-05T07:40:22,241 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a9701d63ca4c4a8fa09a6c10ba3e3f63 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a9701d63ca4c4a8fa09a6c10ba3e3f63 2024-12-05T07:40:22,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8549d45587c845aa8259d71498a6ec9a to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/8549d45587c845aa8259d71498a6ec9a 2024-12-05T07:40:22,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a5334c85f71b443da875b401d3b8102f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/a5334c85f71b443da875b401d3b8102f 2024-12-05T07:40:22,251 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/recovered.edits/468.seqid, newMaxSeqId=468, maxSeqId=1 2024-12-05T07:40:22,252 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. 2024-12-05T07:40:22,252 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 27ae0e4955183fa53f189012e681dac5: Waiting for close lock at 1733384420788Running coprocessor pre-close hooks at 1733384420788Disabling compacts and flushes for region at 1733384420788Disabling writes for close at 1733384420788Obtaining lock to block concurrent updates at 1733384420788Preparing flush snapshotting stores in 27ae0e4955183fa53f189012e681dac5 at 1733384420788Finished memstore snapshotting TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733384420789 (+1 ms)Flushing stores of TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5. at 1733384420790 (+1 ms)Flushing 27ae0e4955183fa53f189012e681dac5/A: creating writer at 1733384420790Flushing 27ae0e4955183fa53f189012e681dac5/A: appending metadata at 1733384420794 (+4 ms)Flushing 27ae0e4955183fa53f189012e681dac5/A: closing flushed file at 1733384420794Flushing 27ae0e4955183fa53f189012e681dac5/B: creating writer at 1733384421209 (+415 ms)Flushing 27ae0e4955183fa53f189012e681dac5/B: appending metadata at 1733384421214 (+5 ms)Flushing 27ae0e4955183fa53f189012e681dac5/B: closing flushed file at 1733384421214Flushing 27ae0e4955183fa53f189012e681dac5/C: creating writer at 1733384421628 (+414 ms)Flushing 27ae0e4955183fa53f189012e681dac5/C: appending metadata at 1733384421631 (+3 ms)Flushing 27ae0e4955183fa53f189012e681dac5/C: closing flushed file at 1733384421631Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cae9618: reopening flushed file at 1733384422040 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47fef4e8: reopening flushed file at 1733384422046 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@367dc973: reopening flushed file at 1733384422053 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 27ae0e4955183fa53f189012e681dac5 in 1271ms, sequenceid=465, compaction requested=true at 1733384422059 (+6 ms)Writing region close event to WAL at 1733384422247 (+188 ms)Running coprocessor post-close hooks at 1733384422252 (+5 ms)Closed at 1733384422252 2024-12-05T07:40:22,257 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=27ae0e4955183fa53f189012e681dac5, regionState=CLOSED 2024-12-05T07:40:22,257 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:22,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:40:22,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-05T07:40:22,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 27ae0e4955183fa53f189012e681dac5, server=fc6cd38557f3,36693,1733384394401 in 1.6350 sec 2024-12-05T07:40:22,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-05T07:40:22,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6600 sec 2024-12-05T07:40:22,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-12-05T07:40:22,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=27ae0e4955183fa53f189012e681dac5, UNASSIGN in 1.6480 sec 2024-12-05T07:40:22,283 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384422282"}]},"ts":"1733384422282"} 2024-12-05T07:40:22,285 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T07:40:22,285 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T07:40:22,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6790 sec 2024-12-05T07:40:22,573 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T07:40:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-05T07:40:22,757 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:22,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T07:40:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:22,768 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T07:40:22,773 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:22,778 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:22,783 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/recovered.edits] 2024-12-05T07:40:22,786 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/2be408ef847a423b973272f338ca186f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/2be408ef847a423b973272f338ca186f 2024-12-05T07:40:22,788 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/2fdc6192cbd0449ea8307b4663d08ab5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/2fdc6192cbd0449ea8307b4663d08ab5 2024-12-05T07:40:22,790 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/77a0d84757bb47cb9a9a5ab65b62aeff to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/A/77a0d84757bb47cb9a9a5ab65b62aeff 2024-12-05T07:40:22,794 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/426e9adfc41247de88a0dc61f87270a5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/426e9adfc41247de88a0dc61f87270a5 2024-12-05T07:40:22,796 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/73db5e7d0c2e4e17b65790b9245a3800 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/73db5e7d0c2e4e17b65790b9245a3800 2024-12-05T07:40:22,798 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b5c0278f43704344bdf07943a0d5b7e5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/B/b5c0278f43704344bdf07943a0d5b7e5 2024-12-05T07:40:22,804 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/03c1d6d01e544eeb9782166e928664b3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/03c1d6d01e544eeb9782166e928664b3 2024-12-05T07:40:22,806 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/36e8b92a498d44d087c459ac761f6870 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/C/36e8b92a498d44d087c459ac761f6870 2024-12-05T07:40:22,810 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/recovered.edits/468.seqid to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5/recovered.edits/468.seqid 2024-12-05T07:40:22,811 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/27ae0e4955183fa53f189012e681dac5 2024-12-05T07:40:22,811 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T07:40:22,817 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:22,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-05T07:40:22,831 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T07:40:22,862 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T07:40:22,864 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:22,864 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T07:40:22,864 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733384422864"}]},"ts":"9223372036854775807"} 2024-12-05T07:40:22,867 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T07:40:22,867 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 27ae0e4955183fa53f189012e681dac5, NAME => 'TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T07:40:22,868 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T07:40:22,868 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733384422868"}]},"ts":"9223372036854775807"} 2024-12-05T07:40:22,871 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T07:40:22,872 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:22,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 110 msec 2024-12-05T07:40:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-05T07:40:22,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T07:40:22,876 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:22,887 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1307116275_22 at /127.0.0.1:45248 [Waiting for operation #333] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fc6cd38557f3:36693-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_480999897_22 at /127.0.0.1:45378 [Waiting for operation #311] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=456 (was 345) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6866 (was 7679) 2024-12-05T07:40:22,896 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=456, ProcessCount=11, AvailableMemoryMB=6865 2024-12-05T07:40:22,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T07:40:22,897 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T07:40:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:22,900 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T07:40:22,900 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:22,900 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-12-05T07:40:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T07:40:22,901 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T07:40:22,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741938_1114 (size=959) 2024-12-05T07:40:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T07:40:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T07:40:23,313 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15 2024-12-05T07:40:23,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741939_1115 (size=53) 2024-12-05T07:40:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T07:40:23,727 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:40:23,727 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 05019411e53a9474ed45d7e906114123, disabling compactions & flushes 2024-12-05T07:40:23,727 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:23,727 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:23,727 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. after waiting 0 ms 2024-12-05T07:40:23,727 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:23,728 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:23,728 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 05019411e53a9474ed45d7e906114123: Waiting for close lock at 1733384423727Disabling compacts and flushes for region at 1733384423727Disabling writes for close at 1733384423727Writing region close event to WAL at 1733384423728 (+1 ms)Closed at 1733384423728 2024-12-05T07:40:23,729 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T07:40:23,730 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733384423729"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733384423729"}]},"ts":"1733384423729"} 2024-12-05T07:40:23,732 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T07:40:23,733 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T07:40:23,733 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384423733"}]},"ts":"1733384423733"} 2024-12-05T07:40:23,735 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T07:40:23,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, ASSIGN}] 2024-12-05T07:40:23,737 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, ASSIGN 2024-12-05T07:40:23,738 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, ASSIGN; state=OFFLINE, location=fc6cd38557f3,36693,1733384394401; forceNewPlan=false, retain=false 2024-12-05T07:40:23,808 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T07:40:23,889 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=05019411e53a9474ed45d7e906114123, regionState=OPENING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:23,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, ASSIGN because future has completed 2024-12-05T07:40:23,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:40:24,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T07:40:24,050 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:24,050 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} 2024-12-05T07:40:24,051 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,051 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:40:24,051 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,051 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,052 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,055 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05019411e53a9474ed45d7e906114123 columnFamilyName A 2024-12-05T07:40:24,055 DEBUG [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:24,056 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(327): Store=05019411e53a9474ed45d7e906114123/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:40:24,056 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,057 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05019411e53a9474ed45d7e906114123 columnFamilyName B 2024-12-05T07:40:24,058 DEBUG [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:24,058 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(327): Store=05019411e53a9474ed45d7e906114123/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:40:24,058 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,060 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05019411e53a9474ed45d7e906114123 columnFamilyName C 2024-12-05T07:40:24,060 DEBUG [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:24,060 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(327): Store=05019411e53a9474ed45d7e906114123/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:40:24,060 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,061 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,061 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,062 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,062 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,063 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T07:40:24,064 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,066 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T07:40:24,067 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 05019411e53a9474ed45d7e906114123; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66994694, jitterRate=-0.001701265573501587}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T07:40:24,067 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:24,068 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 05019411e53a9474ed45d7e906114123: Running coprocessor pre-open hook at 1733384424051Writing region info on filesystem at 1733384424051Initializing all the Stores at 1733384424052 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384424052Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384424052Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384424052Cleaning up temporary data from old regions at 1733384424062 (+10 ms)Running coprocessor post-open hooks at 1733384424067 (+5 ms)Region opened successfully at 1733384424068 (+1 ms) 2024-12-05T07:40:24,069 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., pid=30, masterSystemTime=1733384424043 2024-12-05T07:40:24,070 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:24,070 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:24,071 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=05019411e53a9474ed45d7e906114123, regionState=OPEN, openSeqNum=2, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:24,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:40:24,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-05T07:40:24,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 in 183 msec 2024-12-05T07:40:24,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-05T07:40:24,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, ASSIGN in 340 msec 2024-12-05T07:40:24,079 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T07:40:24,080 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384424079"}]},"ts":"1733384424079"} 2024-12-05T07:40:24,081 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T07:40:24,082 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T07:40:24,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-12-05T07:40:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-05T07:40:25,057 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:25,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcb2494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:25,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:25,059 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:25,060 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:25,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dbd1564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:25,063 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b24555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:25,064 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:25,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:25,070 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T07:40:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fc6cd38557f3,33643,1733384393481 2024-12-05T07:40:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1bf6b1e5 2024-12-05T07:40:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T07:40:25,072 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T07:40:25,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T07:40:25,077 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T07:40:25,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:25,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741940_1116 (size=995) 2024-12-05T07:40:25,531 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-05T07:40:25,532 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-05T07:40:25,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T07:40:25,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, REOPEN/MOVE}] 2024-12-05T07:40:25,542 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, REOPEN/MOVE 2024-12-05T07:40:25,542 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=05019411e53a9474ed45d7e906114123, regionState=CLOSING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:25,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, REOPEN/MOVE because future has completed 2024-12-05T07:40:25,544 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T07:40:25,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:40:25,698 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:25,698 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T07:40:25,698 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 05019411e53a9474ed45d7e906114123, disabling compactions & flushes 2024-12-05T07:40:25,698 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:25,698 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:25,698 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. after waiting 0 ms 2024-12-05T07:40:25,698 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:25,705 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T07:40:25,706 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:25,706 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 05019411e53a9474ed45d7e906114123: Waiting for close lock at 1733384425698Running coprocessor pre-close hooks at 1733384425698Disabling compacts and flushes for region at 1733384425698Disabling writes for close at 1733384425698Writing region close event to WAL at 1733384425699 (+1 ms)Running coprocessor post-close hooks at 1733384425705 (+6 ms)Closed at 1733384425705 2024-12-05T07:40:25,706 WARN [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 05019411e53a9474ed45d7e906114123 to self. 2024-12-05T07:40:25,709 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:25,709 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=05019411e53a9474ed45d7e906114123, regionState=CLOSED 2024-12-05T07:40:25,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:40:25,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-05T07:40:25,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 in 167 msec 2024-12-05T07:40:25,714 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, REOPEN/MOVE; state=CLOSED, location=fc6cd38557f3,36693,1733384394401; forceNewPlan=false, retain=true 2024-12-05T07:40:25,865 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=05019411e53a9474ed45d7e906114123, regionState=OPENING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:25,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, REOPEN/MOVE because future has completed 2024-12-05T07:40:25,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:40:26,026 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,026 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} 2024-12-05T07:40:26,027 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,027 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:40:26,027 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,027 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,032 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,039 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05019411e53a9474ed45d7e906114123 columnFamilyName A 2024-12-05T07:40:26,041 DEBUG [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:26,041 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(327): Store=05019411e53a9474ed45d7e906114123/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:40:26,042 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,043 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05019411e53a9474ed45d7e906114123 columnFamilyName B 2024-12-05T07:40:26,043 DEBUG [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:26,043 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(327): Store=05019411e53a9474ed45d7e906114123/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:40:26,043 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,044 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05019411e53a9474ed45d7e906114123 columnFamilyName C 2024-12-05T07:40:26,044 DEBUG [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:26,044 INFO [StoreOpener-05019411e53a9474ed45d7e906114123-1 {}] regionserver.HStore(327): Store=05019411e53a9474ed45d7e906114123/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:40:26,044 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,045 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,046 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,047 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,047 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,048 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T07:40:26,049 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,049 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 05019411e53a9474ed45d7e906114123; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63014190, jitterRate=-0.06101539731025696}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T07:40:26,049 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,050 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 05019411e53a9474ed45d7e906114123: Running coprocessor pre-open hook at 1733384426027Writing region info on filesystem at 1733384426027Initializing all the Stores at 1733384426028 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384426029 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384426029Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384426029Cleaning up temporary data from old regions at 1733384426047 (+18 ms)Running coprocessor post-open hooks at 1733384426049 (+2 ms)Region opened successfully at 1733384426050 (+1 ms) 2024-12-05T07:40:26,051 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., pid=35, masterSystemTime=1733384426019 2024-12-05T07:40:26,053 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,053 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,053 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=05019411e53a9474ed45d7e906114123, regionState=OPEN, openSeqNum=5, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:40:26,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-12-05T07:40:26,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 in 189 msec 2024-12-05T07:40:26,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-05T07:40:26,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, REOPEN/MOVE in 516 msec 2024-12-05T07:40:26,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-05T07:40:26,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 526 msec 2024-12-05T07:40:26,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 985 msec 2024-12-05T07:40:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-05T07:40:26,068 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:26,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:26,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:26,070 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd3431d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:26,071 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d7d22e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:26,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:26,073 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfa90e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:26,075 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6505c1d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,076 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:26,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:26,076 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6eba2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:26,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:26,077 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da0ed91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:26,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:26,080 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4096a7cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:26,081 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50cd9621, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,082 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:26,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:26,083 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf055da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:26,084 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:26,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e2eefe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:26,086 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:26,086 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42397953, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:26,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:26,088 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aef52c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:26,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:26,090 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d29d05a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:26,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:26,092 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e409a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,093 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,093 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:26,093 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:26,094 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0ba3a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:26,095 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:26,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e0e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ade04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:26,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:26,097 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d69c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:26,099 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441bf5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:26,100 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:26,100 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:26,101 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:26,101 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:26,101 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:26,101 DEBUG [Thread-609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:26,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:26,101 DEBUG [Thread-607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:26,102 DEBUG [Thread-609 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:26,102 DEBUG [Thread-596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:26,102 DEBUG [Thread-609 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-05T07:40:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:26,103 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:26,103 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:26,104 DEBUG [Thread-596 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T07:40:26,104 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:26,104 DEBUG [Thread-596 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:26,104 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:26,104 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:26,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:26,105 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:26,105 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:26,106 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:26,106 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:26,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:26,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:26,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:40:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:40:26,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:40:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:40:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:40:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:40:26,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:40:26,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:40:26,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:40:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,124 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:26,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:40:26,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c38d7b48e57a47a29f4b39053edec8be_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384426124/Put/seqid=0 2024-12-05T07:40:26,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741941_1117 (size=12154) 2024-12-05T07:40:26,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384486173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384486180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384486193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384486194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384486196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T07:40:26,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:26,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:26,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384486306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384486307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384486307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384486316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384486316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T07:40:26,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384486526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384486526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384486527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384486527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384486527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,592 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:26,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c38d7b48e57a47a29f4b39053edec8be_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c38d7b48e57a47a29f4b39053edec8be_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:26,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/17a69ea054fc45cbb00bba0052c0bb95, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:26,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/17a69ea054fc45cbb00bba0052c0bb95 is 175, key is test_row_0/A:col10/1733384426124/Put/seqid=0 2024-12-05T07:40:26,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741942_1118 (size=30955) 2024-12-05T07:40:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T07:40:26,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:26,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:26,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384486847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384486846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384486847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384486846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384486847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:26,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:26,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,018 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/17a69ea054fc45cbb00bba0052c0bb95 2024-12-05T07:40:27,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/5c346ee0a83e430893f636dd30953160 is 50, key is test_row_0/B:col10/1733384426124/Put/seqid=0 2024-12-05T07:40:27,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741943_1119 (size=12001) 2024-12-05T07:40:27,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:27,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:27,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T07:40:27,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:27,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:27,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:27,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384487367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384487366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:27,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:27,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384487367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384487367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:27,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384487369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:27,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:27,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:27,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:27,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/5c346ee0a83e430893f636dd30953160 2024-12-05T07:40:27,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/921a0bf5f32b4719b3828a8338d1ba3b is 50, key is test_row_0/C:col10/1733384426124/Put/seqid=0 2024-12-05T07:40:27,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741944_1120 (size=12001) 2024-12-05T07:40:27,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:27,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:27,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,802 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T07:40:27,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:27,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:27,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:27,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:27,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:27,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:28,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/921a0bf5f32b4719b3828a8338d1ba3b 2024-12-05T07:40:28,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/17a69ea054fc45cbb00bba0052c0bb95 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/17a69ea054fc45cbb00bba0052c0bb95 2024-12-05T07:40:28,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/17a69ea054fc45cbb00bba0052c0bb95, entries=150, sequenceid=17, filesize=30.2 K 2024-12-05T07:40:28,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/5c346ee0a83e430893f636dd30953160 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5c346ee0a83e430893f636dd30953160 2024-12-05T07:40:28,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5c346ee0a83e430893f636dd30953160, entries=150, sequenceid=17, filesize=11.7 K 2024-12-05T07:40:28,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/921a0bf5f32b4719b3828a8338d1ba3b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/921a0bf5f32b4719b3828a8338d1ba3b 2024-12-05T07:40:28,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/921a0bf5f32b4719b3828a8338d1ba3b, entries=150, sequenceid=17, filesize=11.7 K 2024-12-05T07:40:28,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 05019411e53a9474ed45d7e906114123 in 1988ms, sequenceid=17, compaction requested=false 2024-12-05T07:40:28,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:28,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T07:40:28,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:28,123 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T07:40:28,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205886e9c4c73c1405b85421d3f165b9f57_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384426188/Put/seqid=0 2024-12-05T07:40:28,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741945_1121 (size=12154) 2024-12-05T07:40:28,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,192 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205886e9c4c73c1405b85421d3f165b9f57_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205886e9c4c73c1405b85421d3f165b9f57_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/09ba33515bc94aedb3d14daa335b02c1, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:28,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/09ba33515bc94aedb3d14daa335b02c1 is 175, key is test_row_0/A:col10/1733384426188/Put/seqid=0 2024-12-05T07:40:28,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741946_1122 (size=30955) 2024-12-05T07:40:28,238 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/09ba33515bc94aedb3d14daa335b02c1 2024-12-05T07:40:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T07:40:28,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/304873107e3d4c80a9ba009941ca9cbd is 50, key is test_row_0/B:col10/1733384426188/Put/seqid=0 2024-12-05T07:40:28,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741947_1123 (size=12001) 2024-12-05T07:40:28,289 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/304873107e3d4c80a9ba009941ca9cbd 2024-12-05T07:40:28,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/13d5f25317bf4fc89ca85a64f88abbf2 is 50, key is test_row_0/C:col10/1733384426188/Put/seqid=0 2024-12-05T07:40:28,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741948_1124 (size=12001) 2024-12-05T07:40:28,319 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/13d5f25317bf4fc89ca85a64f88abbf2 2024-12-05T07:40:28,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/09ba33515bc94aedb3d14daa335b02c1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/09ba33515bc94aedb3d14daa335b02c1 2024-12-05T07:40:28,348 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/09ba33515bc94aedb3d14daa335b02c1, entries=150, sequenceid=40, filesize=30.2 K 2024-12-05T07:40:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/304873107e3d4c80a9ba009941ca9cbd as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/304873107e3d4c80a9ba009941ca9cbd 2024-12-05T07:40:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,371 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/304873107e3d4c80a9ba009941ca9cbd, entries=150, sequenceid=40, filesize=11.7 K 2024-12-05T07:40:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/13d5f25317bf4fc89ca85a64f88abbf2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/13d5f25317bf4fc89ca85a64f88abbf2 2024-12-05T07:40:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,394 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/13d5f25317bf4fc89ca85a64f88abbf2, entries=150, sequenceid=40, filesize=11.7 K 2024-12-05T07:40:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,395 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 05019411e53a9474ed45d7e906114123 in 272ms, sequenceid=40, compaction requested=false 2024-12-05T07:40:28,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:28,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:28,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-05T07:40:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-05T07:40:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-05T07:40:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2900 sec 2024-12-05T07:40:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.3010 sec 2024-12-05T07:40:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,563 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:28,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:40:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f0b242bffc2147528227ac82d04b0204_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384428556/Put/seqid=0 2024-12-05T07:40:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741949_1125 (size=17034) 2024-12-05T07:40:28,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,604 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f0b242bffc2147528227ac82d04b0204_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f0b242bffc2147528227ac82d04b0204_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,605 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/e73341d2562642cdbeb6bd53a63fcdf2, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:28,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/e73341d2562642cdbeb6bd53a63fcdf2 is 175, key is test_row_0/A:col10/1733384428556/Put/seqid=0 2024-12-05T07:40:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:28,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384488620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384488626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384488628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384488626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384488632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741950_1126 (size=48139) 2024-12-05T07:40:28,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384488736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384488737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384488737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384488737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384488746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,747 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384488946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384488946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384488946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384488946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:28,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384488956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,043 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/e73341d2562642cdbeb6bd53a63fcdf2 2024-12-05T07:40:29,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/8f9a2dea69254029869839ae9aaec7ae is 50, key is test_row_0/B:col10/1733384428556/Put/seqid=0 2024-12-05T07:40:29,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741951_1127 (size=12001) 2024-12-05T07:40:29,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384489256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384489256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384489256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384489267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384489267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/8f9a2dea69254029869839ae9aaec7ae 2024-12-05T07:40:29,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/a7f1e49aa4ad4d31bf6a4072f5449e96 is 50, key is test_row_0/C:col10/1733384428556/Put/seqid=0 2024-12-05T07:40:29,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741952_1128 (size=12001) 2024-12-05T07:40:29,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/a7f1e49aa4ad4d31bf6a4072f5449e96 2024-12-05T07:40:29,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/e73341d2562642cdbeb6bd53a63fcdf2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/e73341d2562642cdbeb6bd53a63fcdf2 2024-12-05T07:40:29,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/e73341d2562642cdbeb6bd53a63fcdf2, entries=250, sequenceid=51, filesize=47.0 K 2024-12-05T07:40:29,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/8f9a2dea69254029869839ae9aaec7ae as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/8f9a2dea69254029869839ae9aaec7ae 2024-12-05T07:40:29,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/8f9a2dea69254029869839ae9aaec7ae, entries=150, sequenceid=51, filesize=11.7 K 2024-12-05T07:40:29,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/a7f1e49aa4ad4d31bf6a4072f5449e96 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/a7f1e49aa4ad4d31bf6a4072f5449e96 2024-12-05T07:40:29,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/a7f1e49aa4ad4d31bf6a4072f5449e96, entries=150, sequenceid=51, filesize=11.7 K 2024-12-05T07:40:29,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 05019411e53a9474ed45d7e906114123 in 990ms, sequenceid=51, compaction requested=true 2024-12-05T07:40:29,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:29,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:29,553 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:29,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:29,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:29,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:29,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:29,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:29,553 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:29,555 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:29,555 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/A is initiating minor compaction (all files) 2024-12-05T07:40:29,555 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/A in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:29,555 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/17a69ea054fc45cbb00bba0052c0bb95, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/09ba33515bc94aedb3d14daa335b02c1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/e73341d2562642cdbeb6bd53a63fcdf2] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=107.5 K 2024-12-05T07:40:29,555 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:29,556 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/17a69ea054fc45cbb00bba0052c0bb95, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/09ba33515bc94aedb3d14daa335b02c1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/e73341d2562642cdbeb6bd53a63fcdf2] 2024-12-05T07:40:29,556 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:29,557 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/B is initiating minor compaction (all files) 2024-12-05T07:40:29,557 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/B in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:29,557 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5c346ee0a83e430893f636dd30953160, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/304873107e3d4c80a9ba009941ca9cbd, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/8f9a2dea69254029869839ae9aaec7ae] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=35.2 K 2024-12-05T07:40:29,558 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c346ee0a83e430893f636dd30953160, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733384426116 2024-12-05T07:40:29,559 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17a69ea054fc45cbb00bba0052c0bb95, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733384426116 2024-12-05T07:40:29,559 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 304873107e3d4c80a9ba009941ca9cbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733384426159 2024-12-05T07:40:29,560 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f9a2dea69254029869839ae9aaec7ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733384428459 2024-12-05T07:40:29,560 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09ba33515bc94aedb3d14daa335b02c1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733384426159 2024-12-05T07:40:29,561 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e73341d2562642cdbeb6bd53a63fcdf2, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733384428457 2024-12-05T07:40:29,579 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#B#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:29,580 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/33cf85908be4474cb57cf0444bdc98e9 is 50, key is test_row_0/B:col10/1733384428556/Put/seqid=0 2024-12-05T07:40:29,587 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:29,603 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205e18218d7c906467c945a6d3bb18f00af_05019411e53a9474ed45d7e906114123 store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:29,606 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205e18218d7c906467c945a6d3bb18f00af_05019411e53a9474ed45d7e906114123, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:29,609 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e18218d7c906467c945a6d3bb18f00af_05019411e53a9474ed45d7e906114123 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:29,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741953_1129 (size=12104) 2024-12-05T07:40:29,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741954_1130 (size=4469) 2024-12-05T07:40:29,651 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/33cf85908be4474cb57cf0444bdc98e9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/33cf85908be4474cb57cf0444bdc98e9 2024-12-05T07:40:29,665 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05019411e53a9474ed45d7e906114123/B of 05019411e53a9474ed45d7e906114123 into 33cf85908be4474cb57cf0444bdc98e9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:29,665 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:29,665 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/B, priority=13, startTime=1733384429553; duration=0sec 2024-12-05T07:40:29,665 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:29,665 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:B 2024-12-05T07:40:29,666 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:29,668 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:29,670 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/C is initiating minor compaction (all files) 2024-12-05T07:40:29,671 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/C in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:29,671 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/921a0bf5f32b4719b3828a8338d1ba3b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/13d5f25317bf4fc89ca85a64f88abbf2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/a7f1e49aa4ad4d31bf6a4072f5449e96] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=35.2 K 2024-12-05T07:40:29,672 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 921a0bf5f32b4719b3828a8338d1ba3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733384426116 2024-12-05T07:40:29,672 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 13d5f25317bf4fc89ca85a64f88abbf2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733384426159 2024-12-05T07:40:29,672 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a7f1e49aa4ad4d31bf6a4072f5449e96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733384428459 2024-12-05T07:40:29,692 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#C#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:29,693 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/e312f988565741deb6551f8278a85cf7 is 50, key is test_row_0/C:col10/1733384428556/Put/seqid=0 2024-12-05T07:40:29,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741955_1131 (size=12104) 2024-12-05T07:40:29,716 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/e312f988565741deb6551f8278a85cf7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/e312f988565741deb6551f8278a85cf7 2024-12-05T07:40:29,734 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05019411e53a9474ed45d7e906114123/C of 05019411e53a9474ed45d7e906114123 into e312f988565741deb6551f8278a85cf7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:29,734 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:29,734 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/C, priority=13, startTime=1733384429553; duration=0sec 2024-12-05T07:40:29,734 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:29,734 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:C 2024-12-05T07:40:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:29,769 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:29,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T07:40:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384489781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384489783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384489784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384489786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384489787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120576764d4b3d41459ba82173642eeb97ff_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384428622/Put/seqid=0 2024-12-05T07:40:29,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741956_1132 (size=12154) 2024-12-05T07:40:29,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384489896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384489897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384489897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384489897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384489897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,046 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#A#compaction#111 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:30,046 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/07043cc1eddd4fccb76cb8f6ef9b7acb is 175, key is test_row_0/A:col10/1733384428556/Put/seqid=0 2024-12-05T07:40:30,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741957_1133 (size=31058) 2024-12-05T07:40:30,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384490106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384490107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384490107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384490107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384490107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:30,250 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120576764d4b3d41459ba82173642eeb97ff_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120576764d4b3d41459ba82173642eeb97ff_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:30,251 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/08022b8f1bc241599899840616255bc7, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:30,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/08022b8f1bc241599899840616255bc7 is 175, key is test_row_0/A:col10/1733384428622/Put/seqid=0 2024-12-05T07:40:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T07:40:30,266 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:30,267 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-05T07:40:30,269 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T07:40:30,269 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:30,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:30,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741958_1134 (size=30955) 2024-12-05T07:40:30,303 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/08022b8f1bc241599899840616255bc7 2024-12-05T07:40:30,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/b6608ed402ce4019ad45f370b92177cc is 50, key is test_row_0/B:col10/1733384428622/Put/seqid=0 2024-12-05T07:40:30,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741959_1135 (size=12001) 2024-12-05T07:40:30,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/b6608ed402ce4019ad45f370b92177cc 2024-12-05T07:40:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T07:40:30,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/ff7a378c1bf44e1e9135e0aed66c1c5a is 50, key is test_row_0/C:col10/1733384428622/Put/seqid=0 2024-12-05T07:40:30,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384490416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384490416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384490416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384490416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384490416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T07:40:30,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:30,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:30,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:30,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741960_1136 (size=12001) 2024-12-05T07:40:30,511 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/07043cc1eddd4fccb76cb8f6ef9b7acb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/07043cc1eddd4fccb76cb8f6ef9b7acb 2024-12-05T07:40:30,520 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05019411e53a9474ed45d7e906114123/A of 05019411e53a9474ed45d7e906114123 into 07043cc1eddd4fccb76cb8f6ef9b7acb(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:30,520 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:30,521 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/A, priority=13, startTime=1733384429553; duration=0sec 2024-12-05T07:40:30,521 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:30,521 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:A 2024-12-05T07:40:30,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T07:40:30,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:30,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:30,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:30,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T07:40:30,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T07:40:30,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:30,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:30,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:30,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/ff7a378c1bf44e1e9135e0aed66c1c5a 2024-12-05T07:40:30,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T07:40:30,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:30,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:30,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:30,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:30,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T07:40:30,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/08022b8f1bc241599899840616255bc7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/08022b8f1bc241599899840616255bc7 2024-12-05T07:40:30,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/08022b8f1bc241599899840616255bc7, entries=150, sequenceid=79, filesize=30.2 K 2024-12-05T07:40:30,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/b6608ed402ce4019ad45f370b92177cc as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/b6608ed402ce4019ad45f370b92177cc 2024-12-05T07:40:30,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/b6608ed402ce4019ad45f370b92177cc, entries=150, sequenceid=79, filesize=11.7 K 2024-12-05T07:40:30,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/ff7a378c1bf44e1e9135e0aed66c1c5a as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/ff7a378c1bf44e1e9135e0aed66c1c5a 2024-12-05T07:40:30,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/ff7a378c1bf44e1e9135e0aed66c1c5a, entries=150, sequenceid=79, filesize=11.7 K 2024-12-05T07:40:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384490926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 05019411e53a9474ed45d7e906114123 in 1161ms, sequenceid=79, compaction requested=false 2024-12-05T07:40:30,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:30,933 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:30,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T07:40:30,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b0ebb618ae06414e8b75abf17fe7fca9_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384430932/Put/seqid=0 2024-12-05T07:40:30,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384490974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741961_1137 (size=14594) 2024-12-05T07:40:30,986 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:30,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384490979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384490980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384490980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:30,996 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b0ebb618ae06414e8b75abf17fe7fca9_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b0ebb618ae06414e8b75abf17fe7fca9_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:30,999 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/fde6f28cc9f34672927370ff7f2da569, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:31,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/fde6f28cc9f34672927370ff7f2da569 is 175, key is test_row_0/A:col10/1733384430932/Put/seqid=0 2024-12-05T07:40:31,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741962_1138 (size=39549) 2024-12-05T07:40:31,044 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/fde6f28cc9f34672927370ff7f2da569 2024-12-05T07:40:31,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T07:40:31,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:31,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/6df1b6c1e4a94a5b9291e86b5104d2a5 is 50, key is test_row_0/B:col10/1733384430932/Put/seqid=0 2024-12-05T07:40:31,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741963_1139 (size=12001) 2024-12-05T07:40:31,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/6df1b6c1e4a94a5b9291e86b5104d2a5 2024-12-05T07:40:31,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384491086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384491096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384491097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384491097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/30c9950f14e645dcaa59ea0dad1ca58c is 50, key is test_row_0/C:col10/1733384430932/Put/seqid=0 2024-12-05T07:40:31,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741964_1140 (size=12001) 2024-12-05T07:40:31,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T07:40:31,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:31,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384491296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384491306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384491306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384491306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T07:40:31,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:31,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T07:40:31,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T07:40:31,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:31,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:31,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/30c9950f14e645dcaa59ea0dad1ca58c 2024-12-05T07:40:31,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384491607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/fde6f28cc9f34672927370ff7f2da569 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/fde6f28cc9f34672927370ff7f2da569 2024-12-05T07:40:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/fde6f28cc9f34672927370ff7f2da569, entries=200, sequenceid=94, filesize=38.6 K 2024-12-05T07:40:31,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/6df1b6c1e4a94a5b9291e86b5104d2a5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/6df1b6c1e4a94a5b9291e86b5104d2a5 2024-12-05T07:40:31,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384491616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384491616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384491617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:31,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/6df1b6c1e4a94a5b9291e86b5104d2a5, entries=150, sequenceid=94, filesize=11.7 K 2024-12-05T07:40:31,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/30c9950f14e645dcaa59ea0dad1ca58c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/30c9950f14e645dcaa59ea0dad1ca58c 2024-12-05T07:40:31,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/30c9950f14e645dcaa59ea0dad1ca58c, entries=150, sequenceid=94, filesize=11.7 K 2024-12-05T07:40:31,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 05019411e53a9474ed45d7e906114123 in 719ms, sequenceid=94, compaction requested=true 2024-12-05T07:40:31,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:31,653 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:31,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:31,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:31,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:31,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:31,653 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:31,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:31,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:31,654 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:31,654 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/A is initiating minor compaction (all files) 2024-12-05T07:40:31,654 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/A in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,654 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/07043cc1eddd4fccb76cb8f6ef9b7acb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/08022b8f1bc241599899840616255bc7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/fde6f28cc9f34672927370ff7f2da569] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=99.2 K 2024-12-05T07:40:31,654 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,654 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/07043cc1eddd4fccb76cb8f6ef9b7acb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/08022b8f1bc241599899840616255bc7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/fde6f28cc9f34672927370ff7f2da569] 2024-12-05T07:40:31,655 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:31,655 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/B is initiating minor compaction (all files) 2024-12-05T07:40:31,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T07:40:31,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,662 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T07:40:31,662 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/B in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,663 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/33cf85908be4474cb57cf0444bdc98e9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/b6608ed402ce4019ad45f370b92177cc, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/6df1b6c1e4a94a5b9291e86b5104d2a5] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=35.3 K 2024-12-05T07:40:31,663 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07043cc1eddd4fccb76cb8f6ef9b7acb, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733384428459 2024-12-05T07:40:31,663 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 33cf85908be4474cb57cf0444bdc98e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733384428459 2024-12-05T07:40:31,663 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08022b8f1bc241599899840616255bc7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733384428622 2024-12-05T07:40:31,664 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting b6608ed402ce4019ad45f370b92177cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733384428622 2024-12-05T07:40:31,664 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting fde6f28cc9f34672927370ff7f2da569, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733384429776 2024-12-05T07:40:31,664 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6df1b6c1e4a94a5b9291e86b5104d2a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733384429776 2024-12-05T07:40:31,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057941c9d3d98847a18578205929875e33_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384430978/Put/seqid=0 2024-12-05T07:40:31,696 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:31,698 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#B#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:31,700 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/24c9737dda72446f841c319ce85a19da is 50, key is test_row_0/B:col10/1733384430932/Put/seqid=0 2024-12-05T07:40:31,707 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412059c5f10c2b04044e186c210b83583c28b_05019411e53a9474ed45d7e906114123 store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:31,709 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412059c5f10c2b04044e186c210b83583c28b_05019411e53a9474ed45d7e906114123, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:31,709 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059c5f10c2b04044e186c210b83583c28b_05019411e53a9474ed45d7e906114123 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:31,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741965_1141 (size=12154) 2024-12-05T07:40:31,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741966_1142 (size=12207) 2024-12-05T07:40:31,790 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/24c9737dda72446f841c319ce85a19da as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/24c9737dda72446f841c319ce85a19da 2024-12-05T07:40:31,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741967_1143 (size=4469) 2024-12-05T07:40:31,799 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05019411e53a9474ed45d7e906114123/B of 05019411e53a9474ed45d7e906114123 into 24c9737dda72446f841c319ce85a19da(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:31,799 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:31,799 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/B, priority=13, startTime=1733384431653; duration=0sec 2024-12-05T07:40:31,799 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:31,799 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:B 2024-12-05T07:40:31,799 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:31,801 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:31,801 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/C is initiating minor compaction (all files) 2024-12-05T07:40:31,801 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/C in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:31,801 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/e312f988565741deb6551f8278a85cf7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/ff7a378c1bf44e1e9135e0aed66c1c5a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/30c9950f14e645dcaa59ea0dad1ca58c] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=35.3 K 2024-12-05T07:40:31,802 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting e312f988565741deb6551f8278a85cf7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733384428459 2024-12-05T07:40:31,803 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting ff7a378c1bf44e1e9135e0aed66c1c5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733384428622 2024-12-05T07:40:31,803 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 30c9950f14e645dcaa59ea0dad1ca58c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733384429776 2024-12-05T07:40:31,814 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#C#compaction#122 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:31,814 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/5cfb176ecfdd4da3bde47d0550a02ee9 is 50, key is test_row_0/C:col10/1733384430932/Put/seqid=0 2024-12-05T07:40:31,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741968_1144 (size=12207) 2024-12-05T07:40:31,853 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/5cfb176ecfdd4da3bde47d0550a02ee9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/5cfb176ecfdd4da3bde47d0550a02ee9 2024-12-05T07:40:31,865 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05019411e53a9474ed45d7e906114123/C of 05019411e53a9474ed45d7e906114123 into 5cfb176ecfdd4da3bde47d0550a02ee9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:31,865 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:31,865 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/C, priority=13, startTime=1733384431653; duration=0sec 2024-12-05T07:40:31,865 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:31,865 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:C 2024-12-05T07:40:31,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:31,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:31,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384491984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:31,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:31,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:32,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384492096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:32,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384492127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384492127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:32,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384492136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384492136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:32,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:32,179 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057941c9d3d98847a18578205929875e33_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057941c9d3d98847a18578205929875e33_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:32,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/6a4ba6201fcb4a2fb421cf1f31d1c6fe, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:32,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/6a4ba6201fcb4a2fb421cf1f31d1c6fe is 175, key is test_row_0/A:col10/1733384430978/Put/seqid=0 2024-12-05T07:40:32,196 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#A#compaction#120 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:32,197 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/ead93b4fe6f9406c9f3cf3d76a1e5b20 is 175, key is test_row_0/A:col10/1733384430932/Put/seqid=0 2024-12-05T07:40:32,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741969_1145 (size=30955) 2024-12-05T07:40:32,200 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/6a4ba6201fcb4a2fb421cf1f31d1c6fe 2024-12-05T07:40:32,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741970_1146 (size=31161) 2024-12-05T07:40:32,248 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/ead93b4fe6f9406c9f3cf3d76a1e5b20 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/ead93b4fe6f9406c9f3cf3d76a1e5b20 2024-12-05T07:40:32,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/5e40efe9e85e445093719e4f04cc7bd8 is 50, key is test_row_0/B:col10/1733384430978/Put/seqid=0 2024-12-05T07:40:32,260 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05019411e53a9474ed45d7e906114123/A of 05019411e53a9474ed45d7e906114123 into ead93b4fe6f9406c9f3cf3d76a1e5b20(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:32,260 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:32,260 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/A, priority=13, startTime=1733384431653; duration=0sec 2024-12-05T07:40:32,260 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:32,260 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:A 2024-12-05T07:40:32,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741971_1147 (size=12001) 2024-12-05T07:40:32,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384492306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T07:40:32,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384492616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:32,703 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/5e40efe9e85e445093719e4f04cc7bd8 2024-12-05T07:40:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/49fdf7487a7248f288d8b1925a967c72 is 50, key is test_row_0/C:col10/1733384430978/Put/seqid=0 2024-12-05T07:40:32,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741972_1148 (size=12001) 2024-12-05T07:40:33,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384493126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:33,130 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/49fdf7487a7248f288d8b1925a967c72 2024-12-05T07:40:33,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/6a4ba6201fcb4a2fb421cf1f31d1c6fe as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/6a4ba6201fcb4a2fb421cf1f31d1c6fe 2024-12-05T07:40:33,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384493136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:33,137 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:33,141 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/6a4ba6201fcb4a2fb421cf1f31d1c6fe, entries=150, sequenceid=117, filesize=30.2 K 2024-12-05T07:40:33,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/5e40efe9e85e445093719e4f04cc7bd8 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5e40efe9e85e445093719e4f04cc7bd8 2024-12-05T07:40:33,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384493146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:33,146 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:33,149 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5e40efe9e85e445093719e4f04cc7bd8, entries=150, sequenceid=117, filesize=11.7 K 2024-12-05T07:40:33,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/49fdf7487a7248f288d8b1925a967c72 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/49fdf7487a7248f288d8b1925a967c72 2024-12-05T07:40:33,155 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/49fdf7487a7248f288d8b1925a967c72, entries=150, sequenceid=117, filesize=11.7 K 2024-12-05T07:40:33,157 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 05019411e53a9474ed45d7e906114123 in 1496ms, sequenceid=117, compaction requested=false 2024-12-05T07:40:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:33,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-05T07:40:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-05T07:40:33,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:33,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T07:40:33,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-05T07:40:33,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8890 sec 2024-12-05T07:40:33,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.8940 sec 2024-12-05T07:40:33,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fff5581269b44771961c325b123423e6_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384433160/Put/seqid=0 2024-12-05T07:40:33,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741973_1149 (size=12304) 2024-12-05T07:40:33,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384493194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:33,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384493195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:33,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384493306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:33,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384493306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:33,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384493516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:33,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384493516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:33,578 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:33,583 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fff5581269b44771961c325b123423e6_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fff5581269b44771961c325b123423e6_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:33,584 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/cb6d047c8e2042e1b8347b0e5c7d9731, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:33,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/cb6d047c8e2042e1b8347b0e5c7d9731 is 175, key is test_row_0/A:col10/1733384433160/Put/seqid=0 2024-12-05T07:40:33,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741974_1150 (size=31105) 2024-12-05T07:40:33,594 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/cb6d047c8e2042e1b8347b0e5c7d9731 2024-12-05T07:40:33,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/9d3b32e2979c4daa8d045583355728ae is 50, key is test_row_0/B:col10/1733384433160/Put/seqid=0 2024-12-05T07:40:33,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741975_1151 (size=12151) 2024-12-05T07:40:33,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/9d3b32e2979c4daa8d045583355728ae 2024-12-05T07:40:33,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/f20f075153c64fab92e05f7f5511d5f4 is 50, key is test_row_0/C:col10/1733384433160/Put/seqid=0 2024-12-05T07:40:33,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741976_1152 (size=12151) 2024-12-05T07:40:33,808 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T07:40:33,808 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T07:40:33,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384493826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:33,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384493826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:34,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/f20f075153c64fab92e05f7f5511d5f4 2024-12-05T07:40:34,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/cb6d047c8e2042e1b8347b0e5c7d9731 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb6d047c8e2042e1b8347b0e5c7d9731 2024-12-05T07:40:34,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb6d047c8e2042e1b8347b0e5c7d9731, entries=150, sequenceid=135, filesize=30.4 K 2024-12-05T07:40:34,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/9d3b32e2979c4daa8d045583355728ae as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9d3b32e2979c4daa8d045583355728ae 2024-12-05T07:40:34,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9d3b32e2979c4daa8d045583355728ae, entries=150, sequenceid=135, filesize=11.9 K 2024-12-05T07:40:34,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/f20f075153c64fab92e05f7f5511d5f4 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f20f075153c64fab92e05f7f5511d5f4 2024-12-05T07:40:34,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f20f075153c64fab92e05f7f5511d5f4, entries=150, sequenceid=135, filesize=11.9 K 2024-12-05T07:40:34,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 05019411e53a9474ed45d7e906114123 in 893ms, sequenceid=135, compaction requested=true 2024-12-05T07:40:34,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:34,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:34,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:34,053 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:34,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:34,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:34,053 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:34,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:34,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:34,055 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:34,055 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:34,055 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/A is initiating minor compaction (all files) 2024-12-05T07:40:34,055 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/B is initiating minor compaction (all files) 2024-12-05T07:40:34,055 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/B in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:34,055 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/A in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:34,055 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/24c9737dda72446f841c319ce85a19da, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5e40efe9e85e445093719e4f04cc7bd8, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9d3b32e2979c4daa8d045583355728ae] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=35.5 K 2024-12-05T07:40:34,055 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/ead93b4fe6f9406c9f3cf3d76a1e5b20, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/6a4ba6201fcb4a2fb421cf1f31d1c6fe, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb6d047c8e2042e1b8347b0e5c7d9731] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=91.0 K 2024-12-05T07:40:34,055 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:34,055 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/ead93b4fe6f9406c9f3cf3d76a1e5b20, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/6a4ba6201fcb4a2fb421cf1f31d1c6fe, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb6d047c8e2042e1b8347b0e5c7d9731] 2024-12-05T07:40:34,055 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 24c9737dda72446f841c319ce85a19da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733384429776 2024-12-05T07:40:34,055 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ead93b4fe6f9406c9f3cf3d76a1e5b20, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733384429776 2024-12-05T07:40:34,056 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e40efe9e85e445093719e4f04cc7bd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733384430971 2024-12-05T07:40:34,056 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a4ba6201fcb4a2fb421cf1f31d1c6fe, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733384430971 2024-12-05T07:40:34,056 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d3b32e2979c4daa8d045583355728ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733384431983 2024-12-05T07:40:34,056 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb6d047c8e2042e1b8347b0e5c7d9731, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733384431983 2024-12-05T07:40:34,062 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:34,064 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#B#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:34,064 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205dc1340f8ff9741389fb1001705b403cd_05019411e53a9474ed45d7e906114123 store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:34,065 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/a91542801b884059b10cf15207a4daf2 is 50, key is test_row_0/B:col10/1733384433160/Put/seqid=0 2024-12-05T07:40:34,066 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205dc1340f8ff9741389fb1001705b403cd_05019411e53a9474ed45d7e906114123, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:34,066 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205dc1340f8ff9741389fb1001705b403cd_05019411e53a9474ed45d7e906114123 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:34,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741977_1153 (size=12459) 2024-12-05T07:40:34,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741978_1154 (size=4469) 2024-12-05T07:40:34,080 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/a91542801b884059b10cf15207a4daf2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a91542801b884059b10cf15207a4daf2 2024-12-05T07:40:34,089 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05019411e53a9474ed45d7e906114123/B of 05019411e53a9474ed45d7e906114123 into a91542801b884059b10cf15207a4daf2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:34,089 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:34,089 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/B, priority=13, startTime=1733384434053; duration=0sec 2024-12-05T07:40:34,089 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:34,089 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:B 2024-12-05T07:40:34,089 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:40:34,091 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:40:34,091 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/C is initiating minor compaction (all files) 2024-12-05T07:40:34,091 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/C in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:34,091 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/5cfb176ecfdd4da3bde47d0550a02ee9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/49fdf7487a7248f288d8b1925a967c72, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f20f075153c64fab92e05f7f5511d5f4] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=35.5 K 2024-12-05T07:40:34,091 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cfb176ecfdd4da3bde47d0550a02ee9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733384429776 2024-12-05T07:40:34,092 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 49fdf7487a7248f288d8b1925a967c72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733384430971 2024-12-05T07:40:34,093 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f20f075153c64fab92e05f7f5511d5f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733384431983 2024-12-05T07:40:34,107 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#C#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:34,107 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/2662c5be72e24e8d972995f73fd140d7 is 50, key is test_row_0/C:col10/1733384433160/Put/seqid=0 2024-12-05T07:40:34,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741979_1155 (size=12459) 2024-12-05T07:40:34,123 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/2662c5be72e24e8d972995f73fd140d7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/2662c5be72e24e8d972995f73fd140d7 2024-12-05T07:40:34,130 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05019411e53a9474ed45d7e906114123/C of 05019411e53a9474ed45d7e906114123 into 2662c5be72e24e8d972995f73fd140d7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:34,130 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:34,130 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/C, priority=13, startTime=1733384434053; duration=0sec 2024-12-05T07:40:34,130 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:34,130 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:C 2024-12-05T07:40:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:34,149 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:34,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-05T07:40:34,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205034d8adf01b6462ba27a19fd37d71f98_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384433191/Put/seqid=0 2024-12-05T07:40:34,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741980_1156 (size=14794) 2024-12-05T07:40:34,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384494187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:34,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384494296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:34,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:34,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384494335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384494335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T07:40:34,436 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:34,436 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:34,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-05T07:40:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T07:40:34,438 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:34,438 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:34,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:34,475 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#A#compaction#128 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:34,475 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/c6164f2a9db444c78c08a0c6e9ba4834 is 175, key is test_row_0/A:col10/1733384433160/Put/seqid=0 2024-12-05T07:40:34,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741981_1157 (size=31413) 2024-12-05T07:40:34,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:34,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384494506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T07:40:34,569 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:34,573 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205034d8adf01b6462ba27a19fd37d71f98_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205034d8adf01b6462ba27a19fd37d71f98_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:34,574 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/2ee59948c468422d934f19984877eb6c, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:34,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/2ee59948c468422d934f19984877eb6c is 175, key is test_row_0/A:col10/1733384433191/Put/seqid=0 2024-12-05T07:40:34,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741982_1158 (size=39749) 2024-12-05T07:40:34,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T07:40:34,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:34,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:34,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:34,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:34,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:34,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T07:40:34,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:34,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:34,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:34,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:34,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T07:40:34,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384494816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:34,886 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/c6164f2a9db444c78c08a0c6e9ba4834 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/c6164f2a9db444c78c08a0c6e9ba4834 2024-12-05T07:40:34,891 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05019411e53a9474ed45d7e906114123/A of 05019411e53a9474ed45d7e906114123 into c6164f2a9db444c78c08a0c6e9ba4834(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:34,892 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:34,892 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/A, priority=13, startTime=1733384434053; duration=0sec 2024-12-05T07:40:34,892 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:34,892 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:A 2024-12-05T07:40:34,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T07:40:34,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:34,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:34,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:34,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:34,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:34,978 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/2ee59948c468422d934f19984877eb6c 2024-12-05T07:40:34,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/a492990864da4d19bd1bcd5c0704f435 is 50, key is test_row_0/B:col10/1733384433191/Put/seqid=0 2024-12-05T07:40:34,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741983_1159 (size=12151) 2024-12-05T07:40:35,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T07:40:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:35,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:35,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T07:40:35,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384495146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:35,147 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:35,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:35,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:35,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:35,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384495155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:35,156 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:35,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:35,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T07:40:35,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:35,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:35,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:35,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:35,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:35,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384495326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:35,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:35,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384495346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:35,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384495346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:35,348 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:35,348 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:35,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T07:40:35,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:35,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:35,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:35,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:35,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:35,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/a492990864da4d19bd1bcd5c0704f435 2024-12-05T07:40:35,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/b23b361c57be4b47acb2a099f8e939c4 is 50, key is test_row_0/C:col10/1733384433191/Put/seqid=0 2024-12-05T07:40:35,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741984_1160 (size=12151) 2024-12-05T07:40:35,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/b23b361c57be4b47acb2a099f8e939c4 2024-12-05T07:40:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/2ee59948c468422d934f19984877eb6c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/2ee59948c468422d934f19984877eb6c 2024-12-05T07:40:35,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/2ee59948c468422d934f19984877eb6c, entries=200, sequenceid=159, filesize=38.8 K 2024-12-05T07:40:35,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/a492990864da4d19bd1bcd5c0704f435 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a492990864da4d19bd1bcd5c0704f435 2024-12-05T07:40:35,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a492990864da4d19bd1bcd5c0704f435, entries=150, sequenceid=159, filesize=11.9 K 2024-12-05T07:40:35,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/b23b361c57be4b47acb2a099f8e939c4 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b23b361c57be4b47acb2a099f8e939c4 2024-12-05T07:40:35,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b23b361c57be4b47acb2a099f8e939c4, entries=150, sequenceid=159, filesize=11.9 K 2024-12-05T07:40:35,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 05019411e53a9474ed45d7e906114123 in 1290ms, sequenceid=159, compaction requested=false 2024-12-05T07:40:35,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:35,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T07:40:35,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:35,510 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T07:40:35,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055b82dc5ba1564396b681d560d1170e2b_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384434167/Put/seqid=0 2024-12-05T07:40:35,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741985_1161 (size=12304) 2024-12-05T07:40:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T07:40:35,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:35,946 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055b82dc5ba1564396b681d560d1170e2b_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055b82dc5ba1564396b681d560d1170e2b_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:35,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/b825daa772e24c63a923a636aed976a3, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:35,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/b825daa772e24c63a923a636aed976a3 is 175, key is test_row_0/A:col10/1733384434167/Put/seqid=0 2024-12-05T07:40:35,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741986_1162 (size=31105) 2024-12-05T07:40:35,952 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/b825daa772e24c63a923a636aed976a3 2024-12-05T07:40:35,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/0a4d99451d2b48a09702046b9f0c04c4 is 50, key is test_row_0/B:col10/1733384434167/Put/seqid=0 2024-12-05T07:40:35,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741987_1163 (size=12151) 2024-12-05T07:40:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:36,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:36,365 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/0a4d99451d2b48a09702046b9f0c04c4 2024-12-05T07:40:36,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/c939ed2d8b544b73a26d91f6294e925b is 50, key is test_row_0/C:col10/1733384434167/Put/seqid=0 2024-12-05T07:40:36,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741988_1164 (size=12151) 2024-12-05T07:40:36,388 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/c939ed2d8b544b73a26d91f6294e925b 2024-12-05T07:40:36,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/b825daa772e24c63a923a636aed976a3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/b825daa772e24c63a923a636aed976a3 2024-12-05T07:40:36,399 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/b825daa772e24c63a923a636aed976a3, entries=150, sequenceid=174, filesize=30.4 K 2024-12-05T07:40:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/0a4d99451d2b48a09702046b9f0c04c4 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/0a4d99451d2b48a09702046b9f0c04c4 2024-12-05T07:40:36,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384496401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:36,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:36,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:36,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:36,408 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/0a4d99451d2b48a09702046b9f0c04c4, entries=150, sequenceid=174, filesize=11.9 K 2024-12-05T07:40:36,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/c939ed2d8b544b73a26d91f6294e925b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/c939ed2d8b544b73a26d91f6294e925b 2024-12-05T07:40:36,413 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/c939ed2d8b544b73a26d91f6294e925b, entries=150, sequenceid=174, filesize=11.9 K 2024-12-05T07:40:36,414 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 05019411e53a9474ed45d7e906114123 in 904ms, sequenceid=174, compaction requested=true 2024-12-05T07:40:36,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:36,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:36,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-05T07:40:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-05T07:40:36,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-05T07:40:36,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9770 sec 2024-12-05T07:40:36,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.9820 sec 2024-12-05T07:40:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:36,518 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:36,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T07:40:36,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fc6b4c9dd8344041b740129c2d5b9ddf_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384436389/Put/seqid=0 2024-12-05T07:40:36,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741989_1165 (size=14794) 2024-12-05T07:40:36,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384496560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:36,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T07:40:36,586 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:36,586 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-05T07:40:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T07:40:36,588 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:36,589 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:36,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:36,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384496675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T07:40:36,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T07:40:36,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:36,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:36,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:36,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:36,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:36,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:36,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384496885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:36,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:36,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T07:40:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:36,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:36,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T07:40:36,936 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:36,940 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fc6b4c9dd8344041b740129c2d5b9ddf_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fc6b4c9dd8344041b740129c2d5b9ddf_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:36,941 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/5c28d0f6fab94a4cb779c3fdd6da7c69, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:36,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/5c28d0f6fab94a4cb779c3fdd6da7c69 is 175, key is test_row_0/A:col10/1733384436389/Put/seqid=0 2024-12-05T07:40:36,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741990_1166 (size=39749) 2024-12-05T07:40:36,947 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/5c28d0f6fab94a4cb779c3fdd6da7c69 2024-12-05T07:40:36,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/ba7824718e474511a9e361618a9e7f24 is 50, key is test_row_0/B:col10/1733384436389/Put/seqid=0 2024-12-05T07:40:36,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741991_1167 (size=12151) 2024-12-05T07:40:37,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T07:40:37,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:37,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:37,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:37,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:37,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:37,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:37,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384497196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:37,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T07:40:37,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:37,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:37,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:37,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:37,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:37,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T07:40:37,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T07:40:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:37,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:37,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384497356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:37,357 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:37,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:37,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:37,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/ba7824718e474511a9e361618a9e7f24 2024-12-05T07:40:37,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384497366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:37,367 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:37,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:37,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/17f9e460425d4cf590c06fef9c4c06c2 is 50, key is test_row_0/C:col10/1733384436389/Put/seqid=0 2024-12-05T07:40:37,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741992_1168 (size=12151) 2024-12-05T07:40:37,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/17f9e460425d4cf590c06fef9c4c06c2 2024-12-05T07:40:37,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/5c28d0f6fab94a4cb779c3fdd6da7c69 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/5c28d0f6fab94a4cb779c3fdd6da7c69 2024-12-05T07:40:37,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/5c28d0f6fab94a4cb779c3fdd6da7c69, entries=200, sequenceid=197, filesize=38.8 K 2024-12-05T07:40:37,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/ba7824718e474511a9e361618a9e7f24 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ba7824718e474511a9e361618a9e7f24 2024-12-05T07:40:37,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ba7824718e474511a9e361618a9e7f24, entries=150, sequenceid=197, filesize=11.9 K 2024-12-05T07:40:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/17f9e460425d4cf590c06fef9c4c06c2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/17f9e460425d4cf590c06fef9c4c06c2 2024-12-05T07:40:37,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/17f9e460425d4cf590c06fef9c4c06c2, entries=150, sequenceid=197, filesize=11.9 K 2024-12-05T07:40:37,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 05019411e53a9474ed45d7e906114123 in 895ms, sequenceid=197, compaction requested=true 2024-12-05T07:40:37,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:37,414 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:37,414 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:37,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:37,415 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:37,415 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:37,415 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/B is initiating minor compaction (all files) 2024-12-05T07:40:37,415 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/A is initiating minor compaction (all files) 2024-12-05T07:40:37,415 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/B in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:37,416 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/A in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:37,416 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a91542801b884059b10cf15207a4daf2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a492990864da4d19bd1bcd5c0704f435, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/0a4d99451d2b48a09702046b9f0c04c4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ba7824718e474511a9e361618a9e7f24] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=47.8 K 2024-12-05T07:40:37,416 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/c6164f2a9db444c78c08a0c6e9ba4834, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/2ee59948c468422d934f19984877eb6c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/b825daa772e24c63a923a636aed976a3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/5c28d0f6fab94a4cb779c3fdd6da7c69] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=138.7 K 2024-12-05T07:40:37,416 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:37,416 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/c6164f2a9db444c78c08a0c6e9ba4834, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/2ee59948c468422d934f19984877eb6c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/b825daa772e24c63a923a636aed976a3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/5c28d0f6fab94a4cb779c3fdd6da7c69] 2024-12-05T07:40:37,416 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a91542801b884059b10cf15207a4daf2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733384431983 2024-12-05T07:40:37,416 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6164f2a9db444c78c08a0c6e9ba4834, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733384431983 2024-12-05T07:40:37,416 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a492990864da4d19bd1bcd5c0704f435, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733384433191 2024-12-05T07:40:37,417 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ee59948c468422d934f19984877eb6c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733384433188 2024-12-05T07:40:37,417 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a4d99451d2b48a09702046b9f0c04c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733384434167 2024-12-05T07:40:37,417 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting ba7824718e474511a9e361618a9e7f24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733384436389 2024-12-05T07:40:37,417 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b825daa772e24c63a923a636aed976a3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733384434167 2024-12-05T07:40:37,424 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c28d0f6fab94a4cb779c3fdd6da7c69, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733384436389 2024-12-05T07:40:37,439 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#B#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:37,440 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/766fddd5008e41f28c5b0306bc29e479 is 50, key is test_row_0/B:col10/1733384436389/Put/seqid=0 2024-12-05T07:40:37,440 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:37,446 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412056fe5f5668ff0405faa81513c58c17c6f_05019411e53a9474ed45d7e906114123 store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:37,449 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412056fe5f5668ff0405faa81513c58c17c6f_05019411e53a9474ed45d7e906114123, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:37,449 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056fe5f5668ff0405faa81513c58c17c6f_05019411e53a9474ed45d7e906114123 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:37,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741994_1170 (size=4469) 2024-12-05T07:40:37,458 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#A#compaction#141 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:37,459 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/445bd25b7fbd428d9c3fab896198282b is 175, key is test_row_0/A:col10/1733384436389/Put/seqid=0 2024-12-05T07:40:37,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741993_1169 (size=12595) 2024-12-05T07:40:37,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741995_1171 (size=31549) 2024-12-05T07:40:37,477 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/445bd25b7fbd428d9c3fab896198282b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/445bd25b7fbd428d9c3fab896198282b 2024-12-05T07:40:37,483 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 05019411e53a9474ed45d7e906114123/A of 05019411e53a9474ed45d7e906114123 into 445bd25b7fbd428d9c3fab896198282b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:37,483 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:37,483 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/A, priority=12, startTime=1733384437414; duration=0sec 2024-12-05T07:40:37,483 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:37,484 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:A 2024-12-05T07:40:37,484 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:37,486 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:37,487 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/C is initiating minor compaction (all files) 2024-12-05T07:40:37,487 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/C in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:37,487 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/2662c5be72e24e8d972995f73fd140d7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b23b361c57be4b47acb2a099f8e939c4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/c939ed2d8b544b73a26d91f6294e925b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/17f9e460425d4cf590c06fef9c4c06c2] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=47.8 K 2024-12-05T07:40:37,487 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2662c5be72e24e8d972995f73fd140d7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1733384431983 2024-12-05T07:40:37,487 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b23b361c57be4b47acb2a099f8e939c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1733384433191 2024-12-05T07:40:37,488 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c939ed2d8b544b73a26d91f6294e925b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733384434167 2024-12-05T07:40:37,488 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17f9e460425d4cf590c06fef9c4c06c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733384436389 2024-12-05T07:40:37,500 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#C#compaction#142 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:37,501 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/e0c98d52b3394de4813dd3ce764d99eb is 50, key is test_row_0/C:col10/1733384436389/Put/seqid=0 2024-12-05T07:40:37,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T07:40:37,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:37,507 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T07:40:37,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741996_1172 (size=12595) 2024-12-05T07:40:37,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120588fbe5547dbc41c6b71b116328ec98d5_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384436554/Put/seqid=0 2024-12-05T07:40:37,523 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/e0c98d52b3394de4813dd3ce764d99eb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/e0c98d52b3394de4813dd3ce764d99eb 2024-12-05T07:40:37,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741997_1173 (size=12304) 2024-12-05T07:40:37,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:37,531 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120588fbe5547dbc41c6b71b116328ec98d5_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120588fbe5547dbc41c6b71b116328ec98d5_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:37,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/71b4064c32fa40b4a546c7769b2d56c0, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:37,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/71b4064c32fa40b4a546c7769b2d56c0 is 175, key is test_row_0/A:col10/1733384436554/Put/seqid=0 2024-12-05T07:40:37,534 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 05019411e53a9474ed45d7e906114123/C of 05019411e53a9474ed45d7e906114123 into e0c98d52b3394de4813dd3ce764d99eb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:37,535 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:37,535 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/C, priority=12, startTime=1733384437414; duration=0sec 2024-12-05T07:40:37,535 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:37,535 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:C 2024-12-05T07:40:37,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741998_1174 (size=31105) 2024-12-05T07:40:37,543 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/71b4064c32fa40b4a546c7769b2d56c0 2024-12-05T07:40:37,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/169e2369bcd0415ba852992c9230e451 is 50, key is test_row_0/B:col10/1733384436554/Put/seqid=0 2024-12-05T07:40:37,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741999_1175 (size=12151) 2024-12-05T07:40:37,661 INFO [master/fc6cd38557f3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T07:40:37,661 INFO [master/fc6cd38557f3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T07:40:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:37,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T07:40:37,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384497765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:37,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:37,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:37,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:37,870 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/766fddd5008e41f28c5b0306bc29e479 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/766fddd5008e41f28c5b0306bc29e479 2024-12-05T07:40:37,876 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 05019411e53a9474ed45d7e906114123/B of 05019411e53a9474ed45d7e906114123 into 766fddd5008e41f28c5b0306bc29e479(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:37,876 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:37,876 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/B, priority=12, startTime=1733384437414; duration=0sec 2024-12-05T07:40:37,877 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:37,877 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:B 2024-12-05T07:40:37,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384497876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:37,958 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/169e2369bcd0415ba852992c9230e451 2024-12-05T07:40:37,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/0b56cfb871af47129255ea3761b62ad7 is 50, key is test_row_0/C:col10/1733384436554/Put/seqid=0 2024-12-05T07:40:37,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742000_1176 (size=12151) 2024-12-05T07:40:38,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384498086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:38,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:38,376 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/0b56cfb871af47129255ea3761b62ad7 2024-12-05T07:40:38,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/71b4064c32fa40b4a546c7769b2d56c0 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/71b4064c32fa40b4a546c7769b2d56c0 2024-12-05T07:40:38,391 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/71b4064c32fa40b4a546c7769b2d56c0, entries=150, sequenceid=211, filesize=30.4 K 2024-12-05T07:40:38,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/169e2369bcd0415ba852992c9230e451 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/169e2369bcd0415ba852992c9230e451 2024-12-05T07:40:38,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:38,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384498396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:38,402 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/169e2369bcd0415ba852992c9230e451, entries=150, sequenceid=211, filesize=11.9 K 2024-12-05T07:40:38,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/0b56cfb871af47129255ea3761b62ad7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/0b56cfb871af47129255ea3761b62ad7 2024-12-05T07:40:38,417 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/0b56cfb871af47129255ea3761b62ad7, entries=150, sequenceid=211, filesize=11.9 K 2024-12-05T07:40:38,418 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 05019411e53a9474ed45d7e906114123 in 911ms, sequenceid=211, compaction requested=false 2024-12-05T07:40:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:38,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-05T07:40:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-05T07:40:38,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-05T07:40:38,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-12-05T07:40:38,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8370 sec 2024-12-05T07:40:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T07:40:38,746 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:38,747 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-05T07:40:38,750 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:38,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T07:40:38,751 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:38,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T07:40:38,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T07:40:38,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:38,904 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T07:40:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:38,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:38,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205bbb2824c16f5498fa9f1b96fea1b7b1c_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384437764/Put/seqid=0 2024-12-05T07:40:38,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742001_1177 (size=12304) 2024-12-05T07:40:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:38,920 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205bbb2824c16f5498fa9f1b96fea1b7b1c_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bbb2824c16f5498fa9f1b96fea1b7b1c_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:38,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/d7d8a670d0d849e286d0a4f21eb3783c, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:38,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/d7d8a670d0d849e286d0a4f21eb3783c is 175, key is test_row_0/A:col10/1733384437764/Put/seqid=0 2024-12-05T07:40:38,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742002_1178 (size=31105) 2024-12-05T07:40:38,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384498934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:38,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:39,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384499045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:39,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:39,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:39,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T07:40:39,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55648 deadline: 1733384499186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:39,191 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:39,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:39,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55684 deadline: 1733384499195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:39,199 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:39,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384499256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:39,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:39,327 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/d7d8a670d0d849e286d0a4f21eb3783c 2024-12-05T07:40:39,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/a93a7b527e74423e93247db19dc45c21 is 50, key is test_row_0/B:col10/1733384437764/Put/seqid=0 2024-12-05T07:40:39,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742003_1179 (size=12151) 2024-12-05T07:40:39,346 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/a93a7b527e74423e93247db19dc45c21 2024-12-05T07:40:39,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/4e462cd5e5404400b6adcdec393f05ce is 50, key is test_row_0/C:col10/1733384437764/Put/seqid=0 2024-12-05T07:40:39,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742004_1180 (size=12151) 2024-12-05T07:40:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T07:40:39,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384499565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:39,765 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/4e462cd5e5404400b6adcdec393f05ce 2024-12-05T07:40:39,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/d7d8a670d0d849e286d0a4f21eb3783c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/d7d8a670d0d849e286d0a4f21eb3783c 2024-12-05T07:40:39,773 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/d7d8a670d0d849e286d0a4f21eb3783c, entries=150, sequenceid=236, filesize=30.4 K 2024-12-05T07:40:39,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/a93a7b527e74423e93247db19dc45c21 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a93a7b527e74423e93247db19dc45c21 2024-12-05T07:40:39,780 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a93a7b527e74423e93247db19dc45c21, entries=150, sequenceid=236, filesize=11.9 K 2024-12-05T07:40:39,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/4e462cd5e5404400b6adcdec393f05ce as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/4e462cd5e5404400b6adcdec393f05ce 2024-12-05T07:40:39,785 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/4e462cd5e5404400b6adcdec393f05ce, entries=150, sequenceid=236, filesize=11.9 K 2024-12-05T07:40:39,786 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 05019411e53a9474ed45d7e906114123 in 882ms, sequenceid=236, compaction requested=true 2024-12-05T07:40:39,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:39,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:39,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-05T07:40:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-05T07:40:39,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-05T07:40:39,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0360 sec 2024-12-05T07:40:39,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.0420 sec 2024-12-05T07:40:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T07:40:39,886 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:39,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-05T07:40:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T07:40:39,888 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:39,889 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:39,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T07:40:40,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T07:40:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:40,041 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T07:40:40,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205216972bf5c7548118fe2ebbf570a06ec_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384438929/Put/seqid=0 2024-12-05T07:40:40,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742005_1181 (size=12304) 2024-12-05T07:40:40,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:40,058 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205216972bf5c7548118fe2ebbf570a06ec_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205216972bf5c7548118fe2ebbf570a06ec_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/3c9fefb5a2f4481b99bc68dd928f6886, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/3c9fefb5a2f4481b99bc68dd928f6886 is 175, key is test_row_0/A:col10/1733384438929/Put/seqid=0 2024-12-05T07:40:40,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742006_1182 (size=31105) 2024-12-05T07:40:40,064 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/3c9fefb5a2f4481b99bc68dd928f6886 2024-12-05T07:40:40,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/7aca80701fa94c22bca5d83fc2c96d53 is 50, key is test_row_0/B:col10/1733384438929/Put/seqid=0 2024-12-05T07:40:40,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742007_1183 (size=12151) 2024-12-05T07:40:40,077 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/7aca80701fa94c22bca5d83fc2c96d53 2024-12-05T07:40:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:40,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:40,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/92dbca2dbf0d4249b67375522ee9e92f is 50, key is test_row_0/C:col10/1733384438929/Put/seqid=0 2024-12-05T07:40:40,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742008_1184 (size=12151) 2024-12-05T07:40:40,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384500151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:40,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T07:40:40,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384500256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:40,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:40,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:40,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384500466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:40,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:40,502 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/92dbca2dbf0d4249b67375522ee9e92f 2024-12-05T07:40:40,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/3c9fefb5a2f4481b99bc68dd928f6886 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/3c9fefb5a2f4481b99bc68dd928f6886 2024-12-05T07:40:40,512 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/3c9fefb5a2f4481b99bc68dd928f6886, entries=150, sequenceid=249, filesize=30.4 K 2024-12-05T07:40:40,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/7aca80701fa94c22bca5d83fc2c96d53 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/7aca80701fa94c22bca5d83fc2c96d53 2024-12-05T07:40:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T07:40:40,517 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/7aca80701fa94c22bca5d83fc2c96d53, entries=150, sequenceid=249, filesize=11.9 K 2024-12-05T07:40:40,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/92dbca2dbf0d4249b67375522ee9e92f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/92dbca2dbf0d4249b67375522ee9e92f 2024-12-05T07:40:40,523 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/92dbca2dbf0d4249b67375522ee9e92f, entries=150, sequenceid=249, filesize=11.9 K 2024-12-05T07:40:40,524 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 05019411e53a9474ed45d7e906114123 in 483ms, sequenceid=249, compaction requested=true 2024-12-05T07:40:40,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:40,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:40,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-05T07:40:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-05T07:40:40,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-05T07:40:40,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 636 msec 2024-12-05T07:40:40,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 641 msec 2024-12-05T07:40:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:40,777 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:40,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T07:40:40,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205662712c97c8046898759422b87ba6331_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384440135/Put/seqid=0 2024-12-05T07:40:40,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742009_1185 (size=14994) 2024-12-05T07:40:40,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:40,798 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205662712c97c8046898759422b87ba6331_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205662712c97c8046898759422b87ba6331_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:40,800 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/62824ccc0b5c4dfab1942123b138c278, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:40,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/62824ccc0b5c4dfab1942123b138c278 is 175, key is test_row_0/A:col10/1733384440135/Put/seqid=0 2024-12-05T07:40:40,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742010_1186 (size=39949) 2024-12-05T07:40:40,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/62824ccc0b5c4dfab1942123b138c278 2024-12-05T07:40:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384500851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:40,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/f414fbb41a5d458ca29c332dcac538a4 is 50, key is test_row_0/B:col10/1733384440135/Put/seqid=0 2024-12-05T07:40:40,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742011_1187 (size=12301) 2024-12-05T07:40:40,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/f414fbb41a5d458ca29c332dcac538a4 2024-12-05T07:40:40,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/aeae0edde11f4b21ba01fbf14aba0e77 is 50, key is test_row_0/C:col10/1733384440135/Put/seqid=0 2024-12-05T07:40:40,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742012_1188 (size=12301) 2024-12-05T07:40:40,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384500966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T07:40:41,026 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:41,027 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-05T07:40:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T07:40:41,029 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:41,030 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:41,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T07:40:41,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:41,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384501176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:41,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:41,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:41,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:41,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/aeae0edde11f4b21ba01fbf14aba0e77 2024-12-05T07:40:41,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/62824ccc0b5c4dfab1942123b138c278 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/62824ccc0b5c4dfab1942123b138c278 2024-12-05T07:40:41,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/62824ccc0b5c4dfab1942123b138c278, entries=200, sequenceid=273, filesize=39.0 K 2024-12-05T07:40:41,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/f414fbb41a5d458ca29c332dcac538a4 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/f414fbb41a5d458ca29c332dcac538a4 2024-12-05T07:40:41,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/f414fbb41a5d458ca29c332dcac538a4, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T07:40:41,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/aeae0edde11f4b21ba01fbf14aba0e77 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/aeae0edde11f4b21ba01fbf14aba0e77 2024-12-05T07:40:41,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:41,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:41,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T07:40:41,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/aeae0edde11f4b21ba01fbf14aba0e77, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T07:40:41,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 05019411e53a9474ed45d7e906114123 in 574ms, sequenceid=273, compaction requested=true 2024-12-05T07:40:41,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:41,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:41,351 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-05T07:40:41,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:41,351 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-05T07:40:41,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:41,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:41,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:41,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:41,353 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61349 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-05T07:40:41,353 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/B is initiating minor compaction (all files) 2024-12-05T07:40:41,353 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/B in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,354 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/766fddd5008e41f28c5b0306bc29e479, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/169e2369bcd0415ba852992c9230e451, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a93a7b527e74423e93247db19dc45c21, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/7aca80701fa94c22bca5d83fc2c96d53, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/f414fbb41a5d458ca29c332dcac538a4] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=59.9 K 2024-12-05T07:40:41,355 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 766fddd5008e41f28c5b0306bc29e479, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733384436389 2024-12-05T07:40:41,355 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164813 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-05T07:40:41,355 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/A is initiating minor compaction (all files) 2024-12-05T07:40:41,355 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/A in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,355 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/445bd25b7fbd428d9c3fab896198282b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/71b4064c32fa40b4a546c7769b2d56c0, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/d7d8a670d0d849e286d0a4f21eb3783c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/3c9fefb5a2f4481b99bc68dd928f6886, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/62824ccc0b5c4dfab1942123b138c278] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=161.0 K 2024-12-05T07:40:41,355 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,356 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/445bd25b7fbd428d9c3fab896198282b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/71b4064c32fa40b4a546c7769b2d56c0, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/d7d8a670d0d849e286d0a4f21eb3783c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/3c9fefb5a2f4481b99bc68dd928f6886, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/62824ccc0b5c4dfab1942123b138c278] 2024-12-05T07:40:41,356 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 169e2369bcd0415ba852992c9230e451, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733384436547 2024-12-05T07:40:41,356 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 445bd25b7fbd428d9c3fab896198282b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733384436389 2024-12-05T07:40:41,356 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a93a7b527e74423e93247db19dc45c21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733384437748 2024-12-05T07:40:41,357 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71b4064c32fa40b4a546c7769b2d56c0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733384436547 2024-12-05T07:40:41,357 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aca80701fa94c22bca5d83fc2c96d53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733384438925 2024-12-05T07:40:41,357 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7d8a670d0d849e286d0a4f21eb3783c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733384437748 2024-12-05T07:40:41,357 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f414fbb41a5d458ca29c332dcac538a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733384440135 2024-12-05T07:40:41,357 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c9fefb5a2f4481b99bc68dd928f6886, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733384438925 2024-12-05T07:40:41,358 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62824ccc0b5c4dfab1942123b138c278, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733384440135 2024-12-05T07:40:41,372 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:41,381 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:41,381 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/c0bcc1bbbf9c4c5984a006211a6ff502 is 50, key is test_row_0/B:col10/1733384440135/Put/seqid=0 2024-12-05T07:40:41,382 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412052149dfab79da40e1bd89d4a8707459cd_05019411e53a9474ed45d7e906114123 store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:41,384 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412052149dfab79da40e1bd89d4a8707459cd_05019411e53a9474ed45d7e906114123, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:41,384 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052149dfab79da40e1bd89d4a8707459cd_05019411e53a9474ed45d7e906114123 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:41,397 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:41,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T07:40:41,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742013_1189 (size=12915) 2024-12-05T07:40:41,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742014_1190 (size=4469) 2024-12-05T07:40:41,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205da992b086ca04c3689faae16730531fc_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384440782/Put/seqid=0 2024-12-05T07:40:41,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742015_1191 (size=14994) 2024-12-05T07:40:41,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384501463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:41,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384501464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:41,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384501486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:41,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:41,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:41,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384501576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384501576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:41,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:41,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:41,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T07:40:41,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:41,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384501786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384501786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:41,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:41,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,814 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#A#compaction#155 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:41,815 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/bad58434e2464c12b312b3cb28e44a12 is 175, key is test_row_0/A:col10/1733384440135/Put/seqid=0 2024-12-05T07:40:41,819 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/c0bcc1bbbf9c4c5984a006211a6ff502 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/c0bcc1bbbf9c4c5984a006211a6ff502 2024-12-05T07:40:41,834 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 05019411e53a9474ed45d7e906114123/B of 05019411e53a9474ed45d7e906114123 into c0bcc1bbbf9c4c5984a006211a6ff502(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:41,834 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:41,834 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/B, priority=11, startTime=1733384441351; duration=0sec 2024-12-05T07:40:41,834 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:41,835 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:B 2024-12-05T07:40:41,835 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-05T07:40:41,838 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:41,839 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61349 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-05T07:40:41,839 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/C is initiating minor compaction (all files) 2024-12-05T07:40:41,839 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/C in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,839 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/e0c98d52b3394de4813dd3ce764d99eb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/0b56cfb871af47129255ea3761b62ad7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/4e462cd5e5404400b6adcdec393f05ce, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/92dbca2dbf0d4249b67375522ee9e92f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/aeae0edde11f4b21ba01fbf14aba0e77] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=59.9 K 2024-12-05T07:40:41,840 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting e0c98d52b3394de4813dd3ce764d99eb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733384436389 2024-12-05T07:40:41,842 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b56cfb871af47129255ea3761b62ad7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733384436547 2024-12-05T07:40:41,843 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e462cd5e5404400b6adcdec393f05ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733384437748 2024-12-05T07:40:41,845 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 92dbca2dbf0d4249b67375522ee9e92f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733384438925 2024-12-05T07:40:41,846 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting aeae0edde11f4b21ba01fbf14aba0e77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733384440135 2024-12-05T07:40:41,846 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205da992b086ca04c3689faae16730531fc_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205da992b086ca04c3689faae16730531fc_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:41,848 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/44f3135ab0c1472fb36ba6550927429b, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:41,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/44f3135ab0c1472fb36ba6550927429b is 175, key is test_row_0/A:col10/1733384440782/Put/seqid=0 2024-12-05T07:40:41,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742016_1192 (size=31869) 2024-12-05T07:40:41,863 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/bad58434e2464c12b312b3cb28e44a12 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/bad58434e2464c12b312b3cb28e44a12 2024-12-05T07:40:41,868 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 05019411e53a9474ed45d7e906114123/A of 05019411e53a9474ed45d7e906114123 into bad58434e2464c12b312b3cb28e44a12(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:41,868 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:41,869 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/A, priority=11, startTime=1733384441351; duration=0sec 2024-12-05T07:40:41,869 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:41,869 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:A 2024-12-05T07:40:41,872 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:41,873 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/b14091fbb60040f6bb59218ef599e50f is 50, key is test_row_0/C:col10/1733384440135/Put/seqid=0 2024-12-05T07:40:41,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742017_1193 (size=39949) 2024-12-05T07:40:41,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742018_1194 (size=12915) 2024-12-05T07:40:41,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:41,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:41,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:41,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:41,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384501996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:41,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:41,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:42,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384502096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:42,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:42,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:42,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384502106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:42,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T07:40:42,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:42,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:42,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/44f3135ab0c1472fb36ba6550927429b 2024-12-05T07:40:42,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/9fafe45c23384465a2ff90c2d538011b is 50, key is test_row_0/B:col10/1733384440782/Put/seqid=0 2024-12-05T07:40:42,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742019_1195 (size=12301) 2024-12-05T07:40:42,294 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/b14091fbb60040f6bb59218ef599e50f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b14091fbb60040f6bb59218ef599e50f 2024-12-05T07:40:42,305 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 05019411e53a9474ed45d7e906114123/C of 05019411e53a9474ed45d7e906114123 into b14091fbb60040f6bb59218ef599e50f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:42,305 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:42,305 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/C, priority=11, startTime=1733384441352; duration=0sec 2024-12-05T07:40:42,305 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:42,305 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:C 2024-12-05T07:40:42,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:42,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:42,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384502606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:42,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384502616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:42,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/9fafe45c23384465a2ff90c2d538011b 2024-12-05T07:40:42,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/163375a513234551a49866f5beacf500 is 50, key is test_row_0/C:col10/1733384440782/Put/seqid=0 2024-12-05T07:40:42,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742020_1196 (size=12301) 2024-12-05T07:40:42,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:42,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:42,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:42,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:43,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384503016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:43,018 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:43,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:43,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:43,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:43,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:43,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:43,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:43,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/163375a513234551a49866f5beacf500 2024-12-05T07:40:43,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/44f3135ab0c1472fb36ba6550927429b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/44f3135ab0c1472fb36ba6550927429b 2024-12-05T07:40:43,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/44f3135ab0c1472fb36ba6550927429b, entries=200, sequenceid=286, filesize=39.0 K 2024-12-05T07:40:43,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/9fafe45c23384465a2ff90c2d538011b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9fafe45c23384465a2ff90c2d538011b 2024-12-05T07:40:43,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9fafe45c23384465a2ff90c2d538011b, entries=150, sequenceid=286, filesize=12.0 K 2024-12-05T07:40:43,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/163375a513234551a49866f5beacf500 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/163375a513234551a49866f5beacf500 2024-12-05T07:40:43,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/163375a513234551a49866f5beacf500, entries=150, sequenceid=286, filesize=12.0 K 2024-12-05T07:40:43,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 05019411e53a9474ed45d7e906114123 in 1731ms, sequenceid=286, compaction requested=false 2024-12-05T07:40:43,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:43,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T07:40:43,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:43,174 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T07:40:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T07:40:43,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057fa7b299b5854158a5d035b06ebd362d_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384441461/Put/seqid=0 2024-12-05T07:40:43,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742021_1197 (size=12454) 2024-12-05T07:40:43,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:43,591 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057fa7b299b5854158a5d035b06ebd362d_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057fa7b299b5854158a5d035b06ebd362d_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:43,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/cb8e7af0147e4e98918debe29c2bdcde, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:43,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/cb8e7af0147e4e98918debe29c2bdcde is 175, key is test_row_0/A:col10/1733384441461/Put/seqid=0 2024-12-05T07:40:43,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742022_1198 (size=31255) 2024-12-05T07:40:43,612 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/cb8e7af0147e4e98918debe29c2bdcde 2024-12-05T07:40:43,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:43,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:43,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/861867f8efdc46e080a5f21be6aaace5 is 50, key is test_row_0/B:col10/1733384441461/Put/seqid=0 2024-12-05T07:40:43,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742023_1199 (size=12301) 2024-12-05T07:40:43,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384503641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:43,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384503641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:43,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384503766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:43,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384503766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:43,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384503976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:43,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384503976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:44,030 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/861867f8efdc46e080a5f21be6aaace5 2024-12-05T07:40:44,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/f6363b416f15414c89f8299c84f3f6b9 is 50, key is test_row_0/C:col10/1733384441461/Put/seqid=0 2024-12-05T07:40:44,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742024_1200 (size=12301) 2024-12-05T07:40:44,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384504285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:44,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384504286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:44,442 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/f6363b416f15414c89f8299c84f3f6b9 2024-12-05T07:40:44,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/cb8e7af0147e4e98918debe29c2bdcde as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb8e7af0147e4e98918debe29c2bdcde 2024-12-05T07:40:44,450 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb8e7af0147e4e98918debe29c2bdcde, entries=150, sequenceid=312, filesize=30.5 K 2024-12-05T07:40:44,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/861867f8efdc46e080a5f21be6aaace5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/861867f8efdc46e080a5f21be6aaace5 2024-12-05T07:40:44,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/861867f8efdc46e080a5f21be6aaace5, entries=150, sequenceid=312, filesize=12.0 K 2024-12-05T07:40:44,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/f6363b416f15414c89f8299c84f3f6b9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f6363b416f15414c89f8299c84f3f6b9 2024-12-05T07:40:44,459 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f6363b416f15414c89f8299c84f3f6b9, entries=150, sequenceid=312, filesize=12.0 K 2024-12-05T07:40:44,460 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 05019411e53a9474ed45d7e906114123 in 1286ms, sequenceid=312, compaction requested=true 2024-12-05T07:40:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-05T07:40:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-05T07:40:44,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-05T07:40:44,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4310 sec 2024-12-05T07:40:44,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 3.4360 sec 2024-12-05T07:40:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:44,798 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:44,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-05T07:40:44,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205bdac35dc0e754cf2a1c87a1ff4efdfa3_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384444797/Put/seqid=0 2024-12-05T07:40:44,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742025_1201 (size=17534) 2024-12-05T07:40:44,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384504833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:44,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384504834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:44,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384504946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:44,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384504946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:44,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:45,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:45,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55662 deadline: 1733384505036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:45,037 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 05019411e53a9474ed45d7e906114123 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:45,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384505156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384505156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T07:40:45,197 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:45,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-05T07:40:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T07:40:45,199 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:45,200 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:45,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:45,210 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:45,213 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205bdac35dc0e754cf2a1c87a1ff4efdfa3_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bdac35dc0e754cf2a1c87a1ff4efdfa3_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:45,214 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/da5e6b0ba6714c2ca6d109c3ff6d3118, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:45,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/da5e6b0ba6714c2ca6d109c3ff6d3118 is 175, key is test_row_0/A:col10/1733384444797/Put/seqid=0 2024-12-05T07:40:45,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742026_1202 (size=48639) 2024-12-05T07:40:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T07:40:45,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T07:40:45,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:45,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:45,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384505466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:45,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384505466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:45,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T07:40:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:45,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:45,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T07:40:45,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/da5e6b0ba6714c2ca6d109c3ff6d3118 2024-12-05T07:40:45,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/ffaeb1c8b12d4ccba2884956324886e7 is 50, key is test_row_0/B:col10/1733384444797/Put/seqid=0 2024-12-05T07:40:45,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742027_1203 (size=12301) 2024-12-05T07:40:45,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T07:40:45,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:45,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:45,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:45,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T07:40:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:45,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T07:40:45,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T07:40:45,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:45,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:45,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:45,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:45,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55618 deadline: 1733384505975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:45,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55670 deadline: 1733384505976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:45,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:40:46,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/ffaeb1c8b12d4ccba2884956324886e7 2024-12-05T07:40:46,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/b75252200c8049ee8580dfd48d41cd9a is 50, key is test_row_0/C:col10/1733384444797/Put/seqid=0 2024-12-05T07:40:46,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742028_1204 (size=12301) 2024-12-05T07:40:46,104 INFO [Thread-607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-607. 2024-12-05T07:40:46,105 DEBUG [Thread-607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:46,105 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,105 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-12-05T07:40:46,105 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:46,105 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,105 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:46,105 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:46,106 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-12-05T07:40:46,106 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:46,106 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,107 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:46,109 INFO [Thread-609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-609. 2024-12-05T07:40:46,110 DEBUG [Thread-609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:46,110 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,110 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:46,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T07:40:46,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:46,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:46,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:46,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:46,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:46,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:46,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T07:40:46,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:46,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T07:40:46,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T07:40:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:46,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:46,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:46,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:46,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:46,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/b75252200c8049ee8580dfd48d41cd9a 2024-12-05T07:40:46,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/da5e6b0ba6714c2ca6d109c3ff6d3118 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/da5e6b0ba6714c2ca6d109c3ff6d3118 2024-12-05T07:40:46,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/da5e6b0ba6714c2ca6d109c3ff6d3118, entries=250, sequenceid=327, filesize=47.5 K 2024-12-05T07:40:46,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/ffaeb1c8b12d4ccba2884956324886e7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ffaeb1c8b12d4ccba2884956324886e7 2024-12-05T07:40:46,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ffaeb1c8b12d4ccba2884956324886e7, entries=150, sequenceid=327, filesize=12.0 K 2024-12-05T07:40:46,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/b75252200c8049ee8580dfd48d41cd9a as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b75252200c8049ee8580dfd48d41cd9a 2024-12-05T07:40:46,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b75252200c8049ee8580dfd48d41cd9a, entries=150, sequenceid=327, filesize=12.0 K 2024-12-05T07:40:46,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 05019411e53a9474ed45d7e906114123 in 1675ms, sequenceid=327, compaction requested=true 2024-12-05T07:40:46,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:46,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:46,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:46,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:46,473 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:46,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:46,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05019411e53a9474ed45d7e906114123:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:46,473 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:46,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:46,474 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:46,474 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/B is initiating minor compaction (all files) 2024-12-05T07:40:46,474 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151712 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:46,474 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/A is initiating minor compaction (all files) 2024-12-05T07:40:46,474 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/B in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:46,474 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/A in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:46,475 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/bad58434e2464c12b312b3cb28e44a12, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/44f3135ab0c1472fb36ba6550927429b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb8e7af0147e4e98918debe29c2bdcde, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/da5e6b0ba6714c2ca6d109c3ff6d3118] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=148.2 K 2024-12-05T07:40:46,475 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/c0bcc1bbbf9c4c5984a006211a6ff502, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9fafe45c23384465a2ff90c2d538011b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/861867f8efdc46e080a5f21be6aaace5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ffaeb1c8b12d4ccba2884956324886e7] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=48.7 K 2024-12-05T07:40:46,475 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:46,475 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/bad58434e2464c12b312b3cb28e44a12, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/44f3135ab0c1472fb36ba6550927429b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb8e7af0147e4e98918debe29c2bdcde, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/da5e6b0ba6714c2ca6d109c3ff6d3118] 2024-12-05T07:40:46,475 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting c0bcc1bbbf9c4c5984a006211a6ff502, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733384440135 2024-12-05T07:40:46,475 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad58434e2464c12b312b3cb28e44a12, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733384440135 2024-12-05T07:40:46,475 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fafe45c23384465a2ff90c2d538011b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384440782 2024-12-05T07:40:46,475 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44f3135ab0c1472fb36ba6550927429b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384440782 2024-12-05T07:40:46,476 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 861867f8efdc46e080a5f21be6aaace5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733384441459 2024-12-05T07:40:46,476 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb8e7af0147e4e98918debe29c2bdcde, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733384441459 2024-12-05T07:40:46,476 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting ffaeb1c8b12d4ccba2884956324886e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733384443627 2024-12-05T07:40:46,476 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting da5e6b0ba6714c2ca6d109c3ff6d3118, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733384443627 2024-12-05T07:40:46,486 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:46,487 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#B#compaction#167 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:46,488 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205dd6f895bc5e64ee29afb38b8cb877efb_05019411e53a9474ed45d7e906114123 store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:46,488 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/7a580fd5389f4780b6989e3830144cc6 is 50, key is test_row_0/B:col10/1733384444797/Put/seqid=0 2024-12-05T07:40:46,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742029_1205 (size=13051) 2024-12-05T07:40:46,533 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205dd6f895bc5e64ee29afb38b8cb877efb_05019411e53a9474ed45d7e906114123, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:46,533 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205dd6f895bc5e64ee29afb38b8cb877efb_05019411e53a9474ed45d7e906114123 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:46,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742030_1206 (size=4469) 2024-12-05T07:40:46,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-05T07:40:46,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:46,581 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T07:40:46,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120550bb769d09394490a8d4a5461a7d4a8a_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384444826/Put/seqid=0 2024-12-05T07:40:46,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742031_1207 (size=12454) 2024-12-05T07:40:46,900 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/7a580fd5389f4780b6989e3830144cc6 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/7a580fd5389f4780b6989e3830144cc6 2024-12-05T07:40:46,908 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 05019411e53a9474ed45d7e906114123/B of 05019411e53a9474ed45d7e906114123 into 7a580fd5389f4780b6989e3830144cc6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:46,908 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:46,908 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/B, priority=12, startTime=1733384446473; duration=0sec 2024-12-05T07:40:46,908 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:46,908 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:B 2024-12-05T07:40:46,908 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:46,910 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:46,910 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 05019411e53a9474ed45d7e906114123/C is initiating minor compaction (all files) 2024-12-05T07:40:46,910 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05019411e53a9474ed45d7e906114123/C in TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:46,910 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b14091fbb60040f6bb59218ef599e50f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/163375a513234551a49866f5beacf500, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f6363b416f15414c89f8299c84f3f6b9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b75252200c8049ee8580dfd48d41cd9a] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp, totalSize=48.7 K 2024-12-05T07:40:46,910 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting b14091fbb60040f6bb59218ef599e50f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733384440135 2024-12-05T07:40:46,911 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 163375a513234551a49866f5beacf500, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384440782 2024-12-05T07:40:46,911 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f6363b416f15414c89f8299c84f3f6b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733384441459 2024-12-05T07:40:46,912 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting b75252200c8049ee8580dfd48d41cd9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733384443627 2024-12-05T07:40:46,921 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#C#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:46,921 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/fc18e66aa7054a0a82acdf6adf7b2c5d is 50, key is test_row_0/C:col10/1733384444797/Put/seqid=0 2024-12-05T07:40:46,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742032_1208 (size=13051) 2024-12-05T07:40:46,939 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05019411e53a9474ed45d7e906114123#A#compaction#168 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:46,939 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/089614bd5080465ebe46db03698a04e8 is 175, key is test_row_0/A:col10/1733384444797/Put/seqid=0 2024-12-05T07:40:46,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742033_1209 (size=32005) 2024-12-05T07:40:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:46,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. as already flushing 2024-12-05T07:40:46,988 INFO [Thread-596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-596. 2024-12-05T07:40:46,988 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-12-05T07:40:46,988 DEBUG [Thread-596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:46,988 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,988 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:46,988 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,988 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:46,988 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:46,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:46,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:46,999 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120550bb769d09394490a8d4a5461a7d4a8a_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120550bb769d09394490a8d4a5461a7d4a8a_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:47,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/c1ec69ad01cc41bf94daacc4d9c241ca, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:47,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/c1ec69ad01cc41bf94daacc4d9c241ca is 175, key is test_row_0/A:col10/1733384444826/Put/seqid=0 2024-12-05T07:40:47,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742034_1210 (size=31255) 2024-12-05T07:40:47,339 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/fc18e66aa7054a0a82acdf6adf7b2c5d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/fc18e66aa7054a0a82acdf6adf7b2c5d 2024-12-05T07:40:47,346 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 05019411e53a9474ed45d7e906114123/C of 05019411e53a9474ed45d7e906114123 into fc18e66aa7054a0a82acdf6adf7b2c5d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:47,346 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:47,346 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/C, priority=12, startTime=1733384446473; duration=0sec 2024-12-05T07:40:47,347 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:47,347 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:C 2024-12-05T07:40:47,351 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/089614bd5080465ebe46db03698a04e8 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/089614bd5080465ebe46db03698a04e8 2024-12-05T07:40:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T07:40:47,358 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 05019411e53a9474ed45d7e906114123/A of 05019411e53a9474ed45d7e906114123 into 089614bd5080465ebe46db03698a04e8(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:47,358 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:47,358 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., storeName=05019411e53a9474ed45d7e906114123/A, priority=12, startTime=1733384446473; duration=0sec 2024-12-05T07:40:47,358 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:47,358 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05019411e53a9474ed45d7e906114123:A 2024-12-05T07:40:47,405 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/c1ec69ad01cc41bf94daacc4d9c241ca 2024-12-05T07:40:47,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/3b4652138cd34a10a04d4d4fa7b20163 is 50, key is test_row_0/B:col10/1733384444826/Put/seqid=0 2024-12-05T07:40:47,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742035_1211 (size=12301) 2024-12-05T07:40:47,818 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/3b4652138cd34a10a04d4d4fa7b20163 2024-12-05T07:40:47,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/08ec5df8961b43c58fefdeb786d2c625 is 50, key is test_row_0/C:col10/1733384444826/Put/seqid=0 2024-12-05T07:40:47,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742036_1212 (size=12301) 2024-12-05T07:40:48,233 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/08ec5df8961b43c58fefdeb786d2c625 2024-12-05T07:40:48,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/c1ec69ad01cc41bf94daacc4d9c241ca as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/c1ec69ad01cc41bf94daacc4d9c241ca 2024-12-05T07:40:48,249 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/c1ec69ad01cc41bf94daacc4d9c241ca, entries=150, sequenceid=348, filesize=30.5 K 2024-12-05T07:40:48,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/3b4652138cd34a10a04d4d4fa7b20163 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/3b4652138cd34a10a04d4d4fa7b20163 2024-12-05T07:40:48,255 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/3b4652138cd34a10a04d4d4fa7b20163, entries=150, sequenceid=348, filesize=12.0 K 2024-12-05T07:40:48,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/08ec5df8961b43c58fefdeb786d2c625 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/08ec5df8961b43c58fefdeb786d2c625 2024-12-05T07:40:48,259 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/08ec5df8961b43c58fefdeb786d2c625, entries=150, sequenceid=348, filesize=12.0 K 2024-12-05T07:40:48,260 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=13.42 KB/13740 for 05019411e53a9474ed45d7e906114123 in 1680ms, sequenceid=348, compaction requested=false 2024-12-05T07:40:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 05019411e53a9474ed45d7e906114123: 2024-12-05T07:40:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-05T07:40:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-05T07:40:48,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-05T07:40:48,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0610 sec 2024-12-05T07:40:48,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.0670 sec 2024-12-05T07:40:49,048 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-12-05T07:40:49,048 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:49,048 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:49,049 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:49,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:49,247 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-12-05T07:40:49,247 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:49,247 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:49,247 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:49,247 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:49,289 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-12-05T07:40:49,289 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:40:49,289 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:49,289 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:49,289 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T07:40:49,377 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:49,377 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T07:40:49,377 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-05T07:40:49,377 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-05T07:40:49,377 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-12-05T07:40:49,377 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-12-05T07:40:49,377 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-12-05T07:40:49,377 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T07:40:49,377 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4390 2024-12-05T07:40:49,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4297 2024-12-05T07:40:49,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T07:40:49,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1922 2024-12-05T07:40:49,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5764 rows 2024-12-05T07:40:49,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1940 2024-12-05T07:40:49,378 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5820 rows 2024-12-05T07:40:49,378 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T07:40:49,378 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T07:40:49,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:49,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:49,379 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:40:49,380 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T07:40:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T07:40:49,386 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384449385"}]},"ts":"1733384449385"} 2024-12-05T07:40:49,387 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T07:40:49,388 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T07:40:49,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T07:40:49,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, UNASSIGN}] 2024-12-05T07:40:49,392 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, UNASSIGN 2024-12-05T07:40:49,393 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=05019411e53a9474ed45d7e906114123, regionState=CLOSING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:49,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, UNASSIGN because future has completed 2024-12-05T07:40:49,395 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T07:40:49,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:40:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T07:40:49,548 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:49,548 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T07:40:49,548 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 05019411e53a9474ed45d7e906114123, disabling compactions & flushes 2024-12-05T07:40:49,548 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:49,548 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:49,548 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. after waiting 0 ms 2024-12-05T07:40:49,548 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:49,548 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 05019411e53a9474ed45d7e906114123 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-05T07:40:49,554 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057ecc4e2df8e944179fe3edb2460ed36b_05019411e53a9474ed45d7e906114123 is 50, key is test_row_0/A:col10/1733384449287/Put/seqid=0 2024-12-05T07:40:49,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742037_1213 (size=9914) 2024-12-05T07:40:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T07:40:49,961 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:49,968 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057ecc4e2df8e944179fe3edb2460ed36b_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ecc4e2df8e944179fe3edb2460ed36b_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:49,970 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/942597e417c3475fb0c1af6bf163e539, store: [table=TestAcidGuarantees family=A region=05019411e53a9474ed45d7e906114123] 2024-12-05T07:40:49,970 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/942597e417c3475fb0c1af6bf163e539 is 175, key is test_row_0/A:col10/1733384449287/Put/seqid=0 2024-12-05T07:40:49,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742038_1214 (size=22561) 2024-12-05T07:40:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T07:40:50,376 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/942597e417c3475fb0c1af6bf163e539 2024-12-05T07:40:50,390 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/4fd2c083417c4972850a167e6275bf82 is 50, key is test_row_0/B:col10/1733384449287/Put/seqid=0 2024-12-05T07:40:50,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742039_1215 (size=9857) 2024-12-05T07:40:50,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T07:40:50,796 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/4fd2c083417c4972850a167e6275bf82 2024-12-05T07:40:50,806 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/b5529906dbd54f7384aeb85d4358ec23 is 50, key is test_row_0/C:col10/1733384449287/Put/seqid=0 2024-12-05T07:40:50,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742040_1216 (size=9857) 2024-12-05T07:40:51,212 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/b5529906dbd54f7384aeb85d4358ec23 2024-12-05T07:40:51,218 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/A/942597e417c3475fb0c1af6bf163e539 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/942597e417c3475fb0c1af6bf163e539 2024-12-05T07:40:51,224 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/942597e417c3475fb0c1af6bf163e539, entries=100, sequenceid=359, filesize=22.0 K 2024-12-05T07:40:51,226 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/B/4fd2c083417c4972850a167e6275bf82 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/4fd2c083417c4972850a167e6275bf82 2024-12-05T07:40:51,233 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/4fd2c083417c4972850a167e6275bf82, entries=100, sequenceid=359, filesize=9.6 K 2024-12-05T07:40:51,234 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/.tmp/C/b5529906dbd54f7384aeb85d4358ec23 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b5529906dbd54f7384aeb85d4358ec23 2024-12-05T07:40:51,241 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b5529906dbd54f7384aeb85d4358ec23, entries=100, sequenceid=359, filesize=9.6 K 2024-12-05T07:40:51,242 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 05019411e53a9474ed45d7e906114123 in 1694ms, sequenceid=359, compaction requested=true 2024-12-05T07:40:51,242 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/17a69ea054fc45cbb00bba0052c0bb95, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/09ba33515bc94aedb3d14daa335b02c1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/e73341d2562642cdbeb6bd53a63fcdf2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/07043cc1eddd4fccb76cb8f6ef9b7acb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/08022b8f1bc241599899840616255bc7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/fde6f28cc9f34672927370ff7f2da569, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/ead93b4fe6f9406c9f3cf3d76a1e5b20, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/6a4ba6201fcb4a2fb421cf1f31d1c6fe, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/c6164f2a9db444c78c08a0c6e9ba4834, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb6d047c8e2042e1b8347b0e5c7d9731, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/2ee59948c468422d934f19984877eb6c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/b825daa772e24c63a923a636aed976a3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/5c28d0f6fab94a4cb779c3fdd6da7c69, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/445bd25b7fbd428d9c3fab896198282b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/71b4064c32fa40b4a546c7769b2d56c0, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/d7d8a670d0d849e286d0a4f21eb3783c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/3c9fefb5a2f4481b99bc68dd928f6886, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/62824ccc0b5c4dfab1942123b138c278, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/bad58434e2464c12b312b3cb28e44a12, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/44f3135ab0c1472fb36ba6550927429b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb8e7af0147e4e98918debe29c2bdcde, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/da5e6b0ba6714c2ca6d109c3ff6d3118] to archive 2024-12-05T07:40:51,243 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:40:51,245 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/17a69ea054fc45cbb00bba0052c0bb95 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/17a69ea054fc45cbb00bba0052c0bb95 2024-12-05T07:40:51,246 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/09ba33515bc94aedb3d14daa335b02c1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/09ba33515bc94aedb3d14daa335b02c1 2024-12-05T07:40:51,248 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/e73341d2562642cdbeb6bd53a63fcdf2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/e73341d2562642cdbeb6bd53a63fcdf2 2024-12-05T07:40:51,249 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/07043cc1eddd4fccb76cb8f6ef9b7acb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/07043cc1eddd4fccb76cb8f6ef9b7acb 2024-12-05T07:40:51,250 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/08022b8f1bc241599899840616255bc7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/08022b8f1bc241599899840616255bc7 2024-12-05T07:40:51,251 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/fde6f28cc9f34672927370ff7f2da569 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/fde6f28cc9f34672927370ff7f2da569 2024-12-05T07:40:51,253 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/ead93b4fe6f9406c9f3cf3d76a1e5b20 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/ead93b4fe6f9406c9f3cf3d76a1e5b20 2024-12-05T07:40:51,254 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/6a4ba6201fcb4a2fb421cf1f31d1c6fe to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/6a4ba6201fcb4a2fb421cf1f31d1c6fe 2024-12-05T07:40:51,256 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/c6164f2a9db444c78c08a0c6e9ba4834 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/c6164f2a9db444c78c08a0c6e9ba4834 2024-12-05T07:40:51,257 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb6d047c8e2042e1b8347b0e5c7d9731 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb6d047c8e2042e1b8347b0e5c7d9731 2024-12-05T07:40:51,258 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/2ee59948c468422d934f19984877eb6c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/2ee59948c468422d934f19984877eb6c 2024-12-05T07:40:51,260 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/b825daa772e24c63a923a636aed976a3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/b825daa772e24c63a923a636aed976a3 2024-12-05T07:40:51,262 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/5c28d0f6fab94a4cb779c3fdd6da7c69 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/5c28d0f6fab94a4cb779c3fdd6da7c69 2024-12-05T07:40:51,264 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/445bd25b7fbd428d9c3fab896198282b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/445bd25b7fbd428d9c3fab896198282b 2024-12-05T07:40:51,265 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/71b4064c32fa40b4a546c7769b2d56c0 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/71b4064c32fa40b4a546c7769b2d56c0 2024-12-05T07:40:51,267 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/d7d8a670d0d849e286d0a4f21eb3783c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/d7d8a670d0d849e286d0a4f21eb3783c 2024-12-05T07:40:51,269 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/3c9fefb5a2f4481b99bc68dd928f6886 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/3c9fefb5a2f4481b99bc68dd928f6886 2024-12-05T07:40:51,270 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/62824ccc0b5c4dfab1942123b138c278 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/62824ccc0b5c4dfab1942123b138c278 2024-12-05T07:40:51,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/bad58434e2464c12b312b3cb28e44a12 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/bad58434e2464c12b312b3cb28e44a12 2024-12-05T07:40:51,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/44f3135ab0c1472fb36ba6550927429b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/44f3135ab0c1472fb36ba6550927429b 2024-12-05T07:40:51,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb8e7af0147e4e98918debe29c2bdcde to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/cb8e7af0147e4e98918debe29c2bdcde 2024-12-05T07:40:51,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/da5e6b0ba6714c2ca6d109c3ff6d3118 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/da5e6b0ba6714c2ca6d109c3ff6d3118 2024-12-05T07:40:51,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5c346ee0a83e430893f636dd30953160, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/304873107e3d4c80a9ba009941ca9cbd, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/33cf85908be4474cb57cf0444bdc98e9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/8f9a2dea69254029869839ae9aaec7ae, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/b6608ed402ce4019ad45f370b92177cc, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/24c9737dda72446f841c319ce85a19da, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/6df1b6c1e4a94a5b9291e86b5104d2a5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5e40efe9e85e445093719e4f04cc7bd8, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a91542801b884059b10cf15207a4daf2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9d3b32e2979c4daa8d045583355728ae, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a492990864da4d19bd1bcd5c0704f435, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/0a4d99451d2b48a09702046b9f0c04c4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/766fddd5008e41f28c5b0306bc29e479, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ba7824718e474511a9e361618a9e7f24, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/169e2369bcd0415ba852992c9230e451, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a93a7b527e74423e93247db19dc45c21, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/7aca80701fa94c22bca5d83fc2c96d53, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/c0bcc1bbbf9c4c5984a006211a6ff502, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/f414fbb41a5d458ca29c332dcac538a4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9fafe45c23384465a2ff90c2d538011b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/861867f8efdc46e080a5f21be6aaace5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ffaeb1c8b12d4ccba2884956324886e7] to archive 2024-12-05T07:40:51,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:40:51,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5c346ee0a83e430893f636dd30953160 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5c346ee0a83e430893f636dd30953160 2024-12-05T07:40:51,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/304873107e3d4c80a9ba009941ca9cbd to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/304873107e3d4c80a9ba009941ca9cbd 2024-12-05T07:40:51,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/33cf85908be4474cb57cf0444bdc98e9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/33cf85908be4474cb57cf0444bdc98e9 2024-12-05T07:40:51,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/8f9a2dea69254029869839ae9aaec7ae to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/8f9a2dea69254029869839ae9aaec7ae 2024-12-05T07:40:51,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/b6608ed402ce4019ad45f370b92177cc to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/b6608ed402ce4019ad45f370b92177cc 2024-12-05T07:40:51,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/24c9737dda72446f841c319ce85a19da to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/24c9737dda72446f841c319ce85a19da 2024-12-05T07:40:51,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/6df1b6c1e4a94a5b9291e86b5104d2a5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/6df1b6c1e4a94a5b9291e86b5104d2a5 2024-12-05T07:40:51,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5e40efe9e85e445093719e4f04cc7bd8 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/5e40efe9e85e445093719e4f04cc7bd8 2024-12-05T07:40:51,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a91542801b884059b10cf15207a4daf2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a91542801b884059b10cf15207a4daf2 2024-12-05T07:40:51,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9d3b32e2979c4daa8d045583355728ae to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9d3b32e2979c4daa8d045583355728ae 2024-12-05T07:40:51,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a492990864da4d19bd1bcd5c0704f435 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a492990864da4d19bd1bcd5c0704f435 2024-12-05T07:40:51,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/0a4d99451d2b48a09702046b9f0c04c4 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/0a4d99451d2b48a09702046b9f0c04c4 2024-12-05T07:40:51,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/766fddd5008e41f28c5b0306bc29e479 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/766fddd5008e41f28c5b0306bc29e479 2024-12-05T07:40:51,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ba7824718e474511a9e361618a9e7f24 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ba7824718e474511a9e361618a9e7f24 2024-12-05T07:40:51,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/169e2369bcd0415ba852992c9230e451 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/169e2369bcd0415ba852992c9230e451 2024-12-05T07:40:51,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a93a7b527e74423e93247db19dc45c21 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/a93a7b527e74423e93247db19dc45c21 2024-12-05T07:40:51,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/7aca80701fa94c22bca5d83fc2c96d53 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/7aca80701fa94c22bca5d83fc2c96d53 2024-12-05T07:40:51,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/c0bcc1bbbf9c4c5984a006211a6ff502 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/c0bcc1bbbf9c4c5984a006211a6ff502 2024-12-05T07:40:51,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/f414fbb41a5d458ca29c332dcac538a4 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/f414fbb41a5d458ca29c332dcac538a4 2024-12-05T07:40:51,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9fafe45c23384465a2ff90c2d538011b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/9fafe45c23384465a2ff90c2d538011b 2024-12-05T07:40:51,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/861867f8efdc46e080a5f21be6aaace5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/861867f8efdc46e080a5f21be6aaace5 2024-12-05T07:40:51,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ffaeb1c8b12d4ccba2884956324886e7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/ffaeb1c8b12d4ccba2884956324886e7 2024-12-05T07:40:51,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/921a0bf5f32b4719b3828a8338d1ba3b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/13d5f25317bf4fc89ca85a64f88abbf2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/e312f988565741deb6551f8278a85cf7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/a7f1e49aa4ad4d31bf6a4072f5449e96, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/ff7a378c1bf44e1e9135e0aed66c1c5a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/5cfb176ecfdd4da3bde47d0550a02ee9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/30c9950f14e645dcaa59ea0dad1ca58c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/49fdf7487a7248f288d8b1925a967c72, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/2662c5be72e24e8d972995f73fd140d7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f20f075153c64fab92e05f7f5511d5f4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b23b361c57be4b47acb2a099f8e939c4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/c939ed2d8b544b73a26d91f6294e925b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/e0c98d52b3394de4813dd3ce764d99eb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/17f9e460425d4cf590c06fef9c4c06c2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/0b56cfb871af47129255ea3761b62ad7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/4e462cd5e5404400b6adcdec393f05ce, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/92dbca2dbf0d4249b67375522ee9e92f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b14091fbb60040f6bb59218ef599e50f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/aeae0edde11f4b21ba01fbf14aba0e77, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/163375a513234551a49866f5beacf500, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f6363b416f15414c89f8299c84f3f6b9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b75252200c8049ee8580dfd48d41cd9a] to archive 2024-12-05T07:40:51,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:40:51,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/921a0bf5f32b4719b3828a8338d1ba3b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/921a0bf5f32b4719b3828a8338d1ba3b 2024-12-05T07:40:51,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/13d5f25317bf4fc89ca85a64f88abbf2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/13d5f25317bf4fc89ca85a64f88abbf2 2024-12-05T07:40:51,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/e312f988565741deb6551f8278a85cf7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/e312f988565741deb6551f8278a85cf7 2024-12-05T07:40:51,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/a7f1e49aa4ad4d31bf6a4072f5449e96 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/a7f1e49aa4ad4d31bf6a4072f5449e96 2024-12-05T07:40:51,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/ff7a378c1bf44e1e9135e0aed66c1c5a to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/ff7a378c1bf44e1e9135e0aed66c1c5a 2024-12-05T07:40:51,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/5cfb176ecfdd4da3bde47d0550a02ee9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/5cfb176ecfdd4da3bde47d0550a02ee9 2024-12-05T07:40:51,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/30c9950f14e645dcaa59ea0dad1ca58c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/30c9950f14e645dcaa59ea0dad1ca58c 2024-12-05T07:40:51,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/49fdf7487a7248f288d8b1925a967c72 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/49fdf7487a7248f288d8b1925a967c72 2024-12-05T07:40:51,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/2662c5be72e24e8d972995f73fd140d7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/2662c5be72e24e8d972995f73fd140d7 2024-12-05T07:40:51,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f20f075153c64fab92e05f7f5511d5f4 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f20f075153c64fab92e05f7f5511d5f4 2024-12-05T07:40:51,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b23b361c57be4b47acb2a099f8e939c4 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b23b361c57be4b47acb2a099f8e939c4 2024-12-05T07:40:51,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/c939ed2d8b544b73a26d91f6294e925b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/c939ed2d8b544b73a26d91f6294e925b 2024-12-05T07:40:51,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/e0c98d52b3394de4813dd3ce764d99eb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/e0c98d52b3394de4813dd3ce764d99eb 2024-12-05T07:40:51,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/17f9e460425d4cf590c06fef9c4c06c2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/17f9e460425d4cf590c06fef9c4c06c2 2024-12-05T07:40:51,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/0b56cfb871af47129255ea3761b62ad7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/0b56cfb871af47129255ea3761b62ad7 2024-12-05T07:40:51,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/4e462cd5e5404400b6adcdec393f05ce to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/4e462cd5e5404400b6adcdec393f05ce 2024-12-05T07:40:51,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/92dbca2dbf0d4249b67375522ee9e92f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/92dbca2dbf0d4249b67375522ee9e92f 2024-12-05T07:40:51,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b14091fbb60040f6bb59218ef599e50f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b14091fbb60040f6bb59218ef599e50f 2024-12-05T07:40:51,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/aeae0edde11f4b21ba01fbf14aba0e77 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/aeae0edde11f4b21ba01fbf14aba0e77 2024-12-05T07:40:51,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/163375a513234551a49866f5beacf500 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/163375a513234551a49866f5beacf500 2024-12-05T07:40:51,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f6363b416f15414c89f8299c84f3f6b9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/f6363b416f15414c89f8299c84f3f6b9 2024-12-05T07:40:51,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b75252200c8049ee8580dfd48d41cd9a to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b75252200c8049ee8580dfd48d41cd9a 2024-12-05T07:40:51,332 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/recovered.edits/362.seqid, newMaxSeqId=362, maxSeqId=4 2024-12-05T07:40:51,333 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. 2024-12-05T07:40:51,333 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 05019411e53a9474ed45d7e906114123: Waiting for close lock at 1733384449548Running coprocessor pre-close hooks at 1733384449548Disabling compacts and flushes for region at 1733384449548Disabling writes for close at 1733384449548Obtaining lock to block concurrent updates at 1733384449548Preparing flush snapshotting stores in 05019411e53a9474ed45d7e906114123 at 1733384449548Finished memstore snapshotting TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733384449549 (+1 ms)Flushing stores of TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123. at 1733384449549Flushing 05019411e53a9474ed45d7e906114123/A: creating writer at 1733384449549Flushing mob file 05019411e53a9474ed45d7e906114123/A: appending metadata at 1733384449554 (+5 ms)Flushing mob file 05019411e53a9474ed45d7e906114123/A: closing flushed file at 1733384449554Flushing 05019411e53a9474ed45d7e906114123/A: appending metadata at 1733384449970 (+416 ms)Flushing 05019411e53a9474ed45d7e906114123/A: closing flushed file at 1733384449970Flushing 05019411e53a9474ed45d7e906114123/B: creating writer at 1733384450384 (+414 ms)Flushing 05019411e53a9474ed45d7e906114123/B: appending metadata at 1733384450389 (+5 ms)Flushing 05019411e53a9474ed45d7e906114123/B: closing flushed file at 1733384450389Flushing 05019411e53a9474ed45d7e906114123/C: creating writer at 1733384450801 (+412 ms)Flushing 05019411e53a9474ed45d7e906114123/C: appending metadata at 1733384450805 (+4 ms)Flushing 05019411e53a9474ed45d7e906114123/C: closing flushed file at 1733384450805Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c8203e8: reopening flushed file at 1733384451217 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@135a44fc: reopening flushed file at 1733384451224 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d196d34: reopening flushed file at 1733384451233 (+9 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 05019411e53a9474ed45d7e906114123 in 1694ms, sequenceid=359, compaction requested=true at 1733384451242 (+9 ms)Writing region close event to WAL at 1733384451328 (+86 ms)Running coprocessor post-close hooks at 1733384451332 (+4 ms)Closed at 1733384451333 (+1 ms) 2024-12-05T07:40:51,334 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,335 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=05019411e53a9474ed45d7e906114123, regionState=CLOSED 2024-12-05T07:40:51,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:40:51,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-05T07:40:51,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 05019411e53a9474ed45d7e906114123, server=fc6cd38557f3,36693,1733384394401 in 1.9420 sec 2024-12-05T07:40:51,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-12-05T07:40:51,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05019411e53a9474ed45d7e906114123, UNASSIGN in 1.9480 sec 2024-12-05T07:40:51,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-05T07:40:51,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9520 sec 2024-12-05T07:40:51,344 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384451344"}]},"ts":"1733384451344"} 2024-12-05T07:40:51,345 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T07:40:51,345 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T07:40:51,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9650 sec 2024-12-05T07:40:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-05T07:40:51,536 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:51,537 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T07:40:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:51,538 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T07:40:51,539 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:51,541 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,543 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/recovered.edits] 2024-12-05T07:40:51,545 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/089614bd5080465ebe46db03698a04e8 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/089614bd5080465ebe46db03698a04e8 2024-12-05T07:40:51,547 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/942597e417c3475fb0c1af6bf163e539 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/942597e417c3475fb0c1af6bf163e539 2024-12-05T07:40:51,548 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/c1ec69ad01cc41bf94daacc4d9c241ca to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/A/c1ec69ad01cc41bf94daacc4d9c241ca 2024-12-05T07:40:51,551 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/3b4652138cd34a10a04d4d4fa7b20163 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/3b4652138cd34a10a04d4d4fa7b20163 2024-12-05T07:40:51,552 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/4fd2c083417c4972850a167e6275bf82 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/4fd2c083417c4972850a167e6275bf82 2024-12-05T07:40:51,554 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/7a580fd5389f4780b6989e3830144cc6 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/B/7a580fd5389f4780b6989e3830144cc6 2024-12-05T07:40:51,557 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/08ec5df8961b43c58fefdeb786d2c625 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/08ec5df8961b43c58fefdeb786d2c625 2024-12-05T07:40:51,558 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b5529906dbd54f7384aeb85d4358ec23 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/b5529906dbd54f7384aeb85d4358ec23 2024-12-05T07:40:51,564 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/fc18e66aa7054a0a82acdf6adf7b2c5d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/C/fc18e66aa7054a0a82acdf6adf7b2c5d 2024-12-05T07:40:51,572 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/recovered.edits/362.seqid to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123/recovered.edits/362.seqid 2024-12-05T07:40:51,573 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,573 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T07:40:51,573 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T07:40:51,574 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T07:40:51,580 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205034d8adf01b6462ba27a19fd37d71f98_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205034d8adf01b6462ba27a19fd37d71f98_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,582 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205216972bf5c7548118fe2ebbf570a06ec_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205216972bf5c7548118fe2ebbf570a06ec_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,584 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120550bb769d09394490a8d4a5461a7d4a8a_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120550bb769d09394490a8d4a5461a7d4a8a_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,585 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055b82dc5ba1564396b681d560d1170e2b_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055b82dc5ba1564396b681d560d1170e2b_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,587 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205662712c97c8046898759422b87ba6331_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205662712c97c8046898759422b87ba6331_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,588 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120576764d4b3d41459ba82173642eeb97ff_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120576764d4b3d41459ba82173642eeb97ff_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,589 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057941c9d3d98847a18578205929875e33_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057941c9d3d98847a18578205929875e33_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,590 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ecc4e2df8e944179fe3edb2460ed36b_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ecc4e2df8e944179fe3edb2460ed36b_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,591 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057fa7b299b5854158a5d035b06ebd362d_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057fa7b299b5854158a5d035b06ebd362d_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,593 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205886e9c4c73c1405b85421d3f165b9f57_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205886e9c4c73c1405b85421d3f165b9f57_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,594 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120588fbe5547dbc41c6b71b116328ec98d5_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120588fbe5547dbc41c6b71b116328ec98d5_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,596 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b0ebb618ae06414e8b75abf17fe7fca9_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b0ebb618ae06414e8b75abf17fe7fca9_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,597 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bbb2824c16f5498fa9f1b96fea1b7b1c_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bbb2824c16f5498fa9f1b96fea1b7b1c_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,600 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bdac35dc0e754cf2a1c87a1ff4efdfa3_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bdac35dc0e754cf2a1c87a1ff4efdfa3_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,602 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c38d7b48e57a47a29f4b39053edec8be_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c38d7b48e57a47a29f4b39053edec8be_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,603 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205da992b086ca04c3689faae16730531fc_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205da992b086ca04c3689faae16730531fc_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,605 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f0b242bffc2147528227ac82d04b0204_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f0b242bffc2147528227ac82d04b0204_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,612 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fc6b4c9dd8344041b740129c2d5b9ddf_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fc6b4c9dd8344041b740129c2d5b9ddf_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,614 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fff5581269b44771961c325b123423e6_05019411e53a9474ed45d7e906114123 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fff5581269b44771961c325b123423e6_05019411e53a9474ed45d7e906114123 2024-12-05T07:40:51,614 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T07:40:51,617 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:51,619 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T07:40:51,621 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T07:40:51,623 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:51,623 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T07:40:51,623 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733384451623"}]},"ts":"9223372036854775807"} 2024-12-05T07:40:51,625 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T07:40:51,625 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 05019411e53a9474ed45d7e906114123, NAME => 'TestAcidGuarantees,,1733384422897.05019411e53a9474ed45d7e906114123.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T07:40:51,625 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T07:40:51,625 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733384451625"}]},"ts":"9223372036854775807"} 2024-12-05T07:40:51,627 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T07:40:51,628 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:51,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-12-05T07:40:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-05T07:40:51,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T07:40:51,647 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:51,654 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=245 (was 238) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/cluster_8fe0b83e-62de-a3e8-93ab-b871591f10b7/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/cluster_8fe0b83e-62de-a3e8-93ab-b871591f10b7/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1307116275_22 at /127.0.0.1:57142 [Waiting for operation #214] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_480999897_22 at /127.0.0.1:57194 [Waiting for operation #191] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_480999897_22 at /127.0.0.1:57184 [Waiting for operation #197] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1307116275_22 at /127.0.0.1:45378 [Waiting for operation #541] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=505 (was 456) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6360 (was 6865) 2024-12-05T07:40:51,663 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=245, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=505, ProcessCount=11, AvailableMemoryMB=6359 2024-12-05T07:40:51,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T07:40:51,664 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T07:40:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T07:40:51,667 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T07:40:51,667 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:51,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-12-05T07:40:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-05T07:40:51,669 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T07:40:51,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742041_1217 (size=959) 2024-12-05T07:40:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-05T07:40:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-05T07:40:52,079 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15 2024-12-05T07:40:52,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742042_1218 (size=53) 2024-12-05T07:40:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-05T07:40:52,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:40:52,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 063dfe74bcf23acd38effeda60a67fa8, disabling compactions & flushes 2024-12-05T07:40:52,492 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:52,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:52,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. after waiting 0 ms 2024-12-05T07:40:52,492 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:52,492 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:52,493 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 063dfe74bcf23acd38effeda60a67fa8: Waiting for close lock at 1733384452492Disabling compacts and flushes for region at 1733384452492Disabling writes for close at 1733384452492Writing region close event to WAL at 1733384452492Closed at 1733384452492 2024-12-05T07:40:52,493 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T07:40:52,494 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733384452493"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733384452493"}]},"ts":"1733384452493"} 2024-12-05T07:40:52,495 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T07:40:52,496 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T07:40:52,496 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384452496"}]},"ts":"1733384452496"} 2024-12-05T07:40:52,497 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T07:40:52,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=063dfe74bcf23acd38effeda60a67fa8, ASSIGN}] 2024-12-05T07:40:52,498 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=063dfe74bcf23acd38effeda60a67fa8, ASSIGN 2024-12-05T07:40:52,499 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=063dfe74bcf23acd38effeda60a67fa8, ASSIGN; state=OFFLINE, location=fc6cd38557f3,36693,1733384394401; forceNewPlan=false, retain=false 2024-12-05T07:40:52,574 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T07:40:52,650 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=063dfe74bcf23acd38effeda60a67fa8, regionState=OPENING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:52,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=063dfe74bcf23acd38effeda60a67fa8, ASSIGN because future has completed 2024-12-05T07:40:52,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:40:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-05T07:40:52,812 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:52,812 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} 2024-12-05T07:40:52,813 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,813 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:40:52,813 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,813 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,815 INFO [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,818 INFO [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 063dfe74bcf23acd38effeda60a67fa8 columnFamilyName A 2024-12-05T07:40:52,818 DEBUG [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:52,819 INFO [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] regionserver.HStore(327): Store=063dfe74bcf23acd38effeda60a67fa8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:40:52,819 INFO [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,821 INFO [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 063dfe74bcf23acd38effeda60a67fa8 columnFamilyName B 2024-12-05T07:40:52,821 DEBUG [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:52,822 INFO [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] regionserver.HStore(327): Store=063dfe74bcf23acd38effeda60a67fa8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:40:52,822 INFO [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,824 INFO [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 063dfe74bcf23acd38effeda60a67fa8 columnFamilyName C 2024-12-05T07:40:52,824 DEBUG [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:40:52,824 INFO [StoreOpener-063dfe74bcf23acd38effeda60a67fa8-1 {}] regionserver.HStore(327): Store=063dfe74bcf23acd38effeda60a67fa8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:40:52,825 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,826 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,826 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,827 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,828 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,828 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T07:40:52,830 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,832 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T07:40:52,833 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 063dfe74bcf23acd38effeda60a67fa8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73916184, jitterRate=0.10143697261810303}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T07:40:52,833 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:52,833 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 063dfe74bcf23acd38effeda60a67fa8: Running coprocessor pre-open hook at 1733384452813Writing region info on filesystem at 1733384452813Initializing all the Stores at 1733384452815 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384452815Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384452815Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384452815Cleaning up temporary data from old regions at 1733384452828 (+13 ms)Running coprocessor post-open hooks at 1733384452833 (+5 ms)Region opened successfully at 1733384452833 2024-12-05T07:40:52,834 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., pid=59, masterSystemTime=1733384452806 2024-12-05T07:40:52,836 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:52,836 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:52,837 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=063dfe74bcf23acd38effeda60a67fa8, regionState=OPEN, openSeqNum=2, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:52,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:40:52,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-05T07:40:52,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 in 186 msec 2024-12-05T07:40:52,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-05T07:40:52,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=063dfe74bcf23acd38effeda60a67fa8, ASSIGN in 344 msec 2024-12-05T07:40:52,843 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T07:40:52,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384452843"}]},"ts":"1733384452843"} 2024-12-05T07:40:52,844 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T07:40:52,845 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T07:40:52,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1810 sec 2024-12-05T07:40:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-05T07:40:53,828 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:53,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25e579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:53,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:53,834 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:53,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:53,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:53,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4312b41c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:53,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:53,837 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4af90268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,838 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:53,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:53,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:53,841 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T07:40:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is fc6cd38557f3,33643,1733384393481 2024-12-05T07:40:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6cd5f801 2024-12-05T07:40:53,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T07:40:53,843 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T07:40:53,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:53,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:53,845 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:53,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289be103, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:53,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:53,847 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:53,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be72708, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:53,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:53,849 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f1c75fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:53,851 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:53,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60a57ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:53,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:53,853 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d6870c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:53,854 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@687573c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:53,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:53,856 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4611578b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:53,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:53,857 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:53,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b5e856, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,858 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:53,858 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:53,859 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2625dce0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:53,860 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f346810, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:53,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:53,862 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:53,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:53,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a734806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:53,864 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:53,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e48b408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:53,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:53,866 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111b4688, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:53,867 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65403200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:53,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:53,869 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688037bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:53,871 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:53,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6efcaeeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:53,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:53,873 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d939cac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:53,874 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:53,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca81dbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5621a0fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:40:53,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:40:53,876 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:40:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:40:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:40:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7244b721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:40:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:40:53,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:40:53,877 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:40:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1feae9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:40:53,878 DEBUG [Thread-1035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:53,878 DEBUG [Thread-1029 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:53,878 DEBUG [Thread-1037 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:53,879 DEBUG [Thread-1046 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:53,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:53,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:53,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:53,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:53,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:53,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:53,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:53,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:53,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:53,888 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:53,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:40:53,895 DEBUG [Thread-1031 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:53,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:53,898 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:53,903 DEBUG [Thread-1040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:53,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:53,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:53,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:53,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:53,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384513908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:53,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384513909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:53,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384513910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:53,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:53,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384513911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:53,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:53,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/6b591e9affa748f39e8f9cba4df6f343 is 50, key is test_row_0/A:col10/1733384453884/Put/seqid=0 2024-12-05T07:40:53,923 DEBUG [Thread-1048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:53,923 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:53,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:53,924 DEBUG [Thread-1044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:53,924 DEBUG [Thread-1042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:53,925 DEBUG [Thread-1044 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:53,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:53,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:53,925 DEBUG [Thread-1044 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-05T07:40:53,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:53,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:53,927 DEBUG [Thread-1042 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T07:40:53,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:53,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:53,929 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:53,930 DEBUG [Thread-1033 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:40:53,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742043_1219 (size=12001) 2024-12-05T07:40:53,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:53,930 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:53,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:40:53,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:40:53,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:40:53,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:40:53,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:53,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384513935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:53,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:53,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/6b591e9affa748f39e8f9cba4df6f343 2024-12-05T07:40:53,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/d6bc644c8e4c4f6f964302f6f8f9f0c4 is 50, key is test_row_0/B:col10/1733384453884/Put/seqid=0 2024-12-05T07:40:53,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742044_1220 (size=12001) 2024-12-05T07:40:53,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/d6bc644c8e4c4f6f964302f6f8f9f0c4 2024-12-05T07:40:54,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384514016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384514016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384514016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384514016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/80cf0af12d7e4f70ac16b0c2cd5d8e35 is 50, key is test_row_0/C:col10/1733384453884/Put/seqid=0 2024-12-05T07:40:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T07:40:54,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742045_1221 (size=12001) 2024-12-05T07:40:54,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384514046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:54,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:54,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384514226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384514226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384514226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384514226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:54,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:54,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T07:40:54,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384514256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:54,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:54,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/80cf0af12d7e4f70ac16b0c2cd5d8e35 2024-12-05T07:40:54,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/6b591e9affa748f39e8f9cba4df6f343 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/6b591e9affa748f39e8f9cba4df6f343 2024-12-05T07:40:54,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/6b591e9affa748f39e8f9cba4df6f343, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T07:40:54,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/d6bc644c8e4c4f6f964302f6f8f9f0c4 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/d6bc644c8e4c4f6f964302f6f8f9f0c4 2024-12-05T07:40:54,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/d6bc644c8e4c4f6f964302f6f8f9f0c4, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T07:40:54,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/80cf0af12d7e4f70ac16b0c2cd5d8e35 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/80cf0af12d7e4f70ac16b0c2cd5d8e35 2024-12-05T07:40:54,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/80cf0af12d7e4f70ac16b0c2cd5d8e35, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T07:40:54,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 063dfe74bcf23acd38effeda60a67fa8 in 568ms, sequenceid=13, compaction requested=false 2024-12-05T07:40:54,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:40:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:54,538 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:54,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T07:40:54,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:54,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/feda5ef151f44f0c85e140ac6f354b5e is 50, key is test_row_0/A:col10/1733384454538/Put/seqid=0 2024-12-05T07:40:54,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:54,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742046_1222 (size=14341) 2024-12-05T07:40:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384514544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384514544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384514545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384514544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T07:40:54,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384514566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384514656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384514656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384514656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384514657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:54,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:54,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:54,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:54,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384514866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384514866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384514866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384514866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:54,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/feda5ef151f44f0c85e140ac6f354b5e 2024-12-05T07:40:54,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/29c30f6ae610498b8d536e5814237034 is 50, key is test_row_0/B:col10/1733384454538/Put/seqid=0 2024-12-05T07:40:54,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742047_1223 (size=12001) 2024-12-05T07:40:55,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:55,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T07:40:55,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384515076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:55,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:55,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:55,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384515176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:55,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384515176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:55,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384515177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384515177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:55,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:55,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:55,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/29c30f6ae610498b8d536e5814237034 2024-12-05T07:40:55,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/27c40958add64b4e9aae431d8bad6c8a is 50, key is test_row_0/C:col10/1733384454538/Put/seqid=0 2024-12-05T07:40:55,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742048_1224 (size=12001) 2024-12-05T07:40:55,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:55,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:55,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:55,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:55,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384515686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:55,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384515686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:55,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:55,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384515686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384515686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:55,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:55,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:55,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:40:55,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/27c40958add64b4e9aae431d8bad6c8a 2024-12-05T07:40:55,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/feda5ef151f44f0c85e140ac6f354b5e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/feda5ef151f44f0c85e140ac6f354b5e 2024-12-05T07:40:55,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/feda5ef151f44f0c85e140ac6f354b5e, entries=200, sequenceid=41, filesize=14.0 K 2024-12-05T07:40:55,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/29c30f6ae610498b8d536e5814237034 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/29c30f6ae610498b8d536e5814237034 2024-12-05T07:40:55,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/29c30f6ae610498b8d536e5814237034, entries=150, sequenceid=41, filesize=11.7 K 2024-12-05T07:40:55,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/27c40958add64b4e9aae431d8bad6c8a as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/27c40958add64b4e9aae431d8bad6c8a 2024-12-05T07:40:55,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/27c40958add64b4e9aae431d8bad6c8a, entries=150, sequenceid=41, filesize=11.7 K 2024-12-05T07:40:55,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 063dfe74bcf23acd38effeda60a67fa8 in 1259ms, sequenceid=41, compaction requested=false 2024-12-05T07:40:55,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:40:55,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T07:40:55,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:55,925 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-05T07:40:55,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/695519345e764cb7917d5072d2928227 is 50, key is test_row_0/A:col10/1733384454542/Put/seqid=0 2024-12-05T07:40:55,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742049_1225 (size=9657) 2024-12-05T07:40:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T07:40:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:56,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:56,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384516161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:56,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384516266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:56,348 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/695519345e764cb7917d5072d2928227 2024-12-05T07:40:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/bffcd5c3434a48fc8e7f27541a4e7d8a is 50, key is test_row_0/B:col10/1733384454542/Put/seqid=0 2024-12-05T07:40:56,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742050_1226 (size=9657) 2024-12-05T07:40:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384516475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:56,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384516696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:56,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384516696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:56,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384516696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:56,698 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:56,698 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,698 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:56,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384516706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:56,707 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:56,775 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/bffcd5c3434a48fc8e7f27541a4e7d8a 2024-12-05T07:40:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/dc06700c59e54ffba9b8760d88b62e3e is 50, key is test_row_0/C:col10/1733384454542/Put/seqid=0 2024-12-05T07:40:56,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384516786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:56,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:56,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742051_1227 (size=9657) 2024-12-05T07:40:56,794 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/dc06700c59e54ffba9b8760d88b62e3e 2024-12-05T07:40:56,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/695519345e764cb7917d5072d2928227 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/695519345e764cb7917d5072d2928227 2024-12-05T07:40:56,812 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/695519345e764cb7917d5072d2928227, entries=100, sequenceid=49, filesize=9.4 K 2024-12-05T07:40:56,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/bffcd5c3434a48fc8e7f27541a4e7d8a as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/bffcd5c3434a48fc8e7f27541a4e7d8a 2024-12-05T07:40:56,822 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/bffcd5c3434a48fc8e7f27541a4e7d8a, entries=100, sequenceid=49, filesize=9.4 K 2024-12-05T07:40:56,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/dc06700c59e54ffba9b8760d88b62e3e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/dc06700c59e54ffba9b8760d88b62e3e 2024-12-05T07:40:56,841 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/dc06700c59e54ffba9b8760d88b62e3e, entries=100, sequenceid=49, filesize=9.4 K 2024-12-05T07:40:56,843 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 063dfe74bcf23acd38effeda60a67fa8 in 918ms, sequenceid=49, compaction requested=true 2024-12-05T07:40:56,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:40:56,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:56,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-05T07:40:56,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-05T07:40:56,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-05T07:40:56,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9160 sec 2024-12-05T07:40:56,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.9250 sec 2024-12-05T07:40:56,854 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T07:40:57,315 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:57,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-12-05T07:40:57,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/44177e9f20ad4563a2df4604913507eb is 50, key is test_row_0/A:col10/1733384456158/Put/seqid=0 2024-12-05T07:40:57,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384517328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:57,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:57,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742052_1228 (size=12001) 2024-12-05T07:40:57,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/44177e9f20ad4563a2df4604913507eb 2024-12-05T07:40:57,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/904997487d3a42e38dca49c6b5dbafb5 is 50, key is test_row_0/B:col10/1733384456158/Put/seqid=0 2024-12-05T07:40:57,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742053_1229 (size=12001) 2024-12-05T07:40:57,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384517436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:57,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384517645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:57,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:57,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:57,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:57,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/904997487d3a42e38dca49c6b5dbafb5 2024-12-05T07:40:57,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/2f372829774f4da69bb3b2bf5e606772 is 50, key is test_row_0/C:col10/1733384456158/Put/seqid=0 2024-12-05T07:40:57,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742054_1230 (size=12001) 2024-12-05T07:40:57,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384517956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:57,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:58,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T07:40:58,096 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:40:58,097 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:40:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-05T07:40:58,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T07:40:58,098 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:40:58,098 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:40:58,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:40:58,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/2f372829774f4da69bb3b2bf5e606772 2024-12-05T07:40:58,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/44177e9f20ad4563a2df4604913507eb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/44177e9f20ad4563a2df4604913507eb 2024-12-05T07:40:58,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/44177e9f20ad4563a2df4604913507eb, entries=150, sequenceid=78, filesize=11.7 K 2024-12-05T07:40:58,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/904997487d3a42e38dca49c6b5dbafb5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/904997487d3a42e38dca49c6b5dbafb5 2024-12-05T07:40:58,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/904997487d3a42e38dca49c6b5dbafb5, entries=150, sequenceid=78, filesize=11.7 K 2024-12-05T07:40:58,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/2f372829774f4da69bb3b2bf5e606772 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/2f372829774f4da69bb3b2bf5e606772 2024-12-05T07:40:58,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/2f372829774f4da69bb3b2bf5e606772, entries=150, sequenceid=78, filesize=11.7 K 2024-12-05T07:40:58,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 063dfe74bcf23acd38effeda60a67fa8 in 875ms, sequenceid=78, compaction requested=true 2024-12-05T07:40:58,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:40:58,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:40:58,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:58,190 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:58,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:40:58,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:58,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:40:58,191 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:58,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:58,192 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48000 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:58,192 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/A is initiating minor compaction (all files) 2024-12-05T07:40:58,192 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/A in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:58,192 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/6b591e9affa748f39e8f9cba4df6f343, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/feda5ef151f44f0c85e140ac6f354b5e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/695519345e764cb7917d5072d2928227, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/44177e9f20ad4563a2df4604913507eb] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=46.9 K 2024-12-05T07:40:58,192 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:58,192 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/B is initiating minor compaction (all files) 2024-12-05T07:40:58,192 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b591e9affa748f39e8f9cba4df6f343, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733384453884 2024-12-05T07:40:58,192 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/B in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:58,193 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/d6bc644c8e4c4f6f964302f6f8f9f0c4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/29c30f6ae610498b8d536e5814237034, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/bffcd5c3434a48fc8e7f27541a4e7d8a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/904997487d3a42e38dca49c6b5dbafb5] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=44.6 K 2024-12-05T07:40:58,193 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting d6bc644c8e4c4f6f964302f6f8f9f0c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733384453884 2024-12-05T07:40:58,193 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting feda5ef151f44f0c85e140ac6f354b5e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733384453905 2024-12-05T07:40:58,193 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 29c30f6ae610498b8d536e5814237034, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733384453905 2024-12-05T07:40:58,193 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 695519345e764cb7917d5072d2928227, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733384454542 2024-12-05T07:40:58,194 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting bffcd5c3434a48fc8e7f27541a4e7d8a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733384454542 2024-12-05T07:40:58,194 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44177e9f20ad4563a2df4604913507eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733384456144 2024-12-05T07:40:58,194 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 904997487d3a42e38dca49c6b5dbafb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733384456144 2024-12-05T07:40:58,205 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#B#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:58,205 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/a0a184ba5b71452b92f9e3b908b5fffc is 50, key is test_row_0/B:col10/1733384456158/Put/seqid=0 2024-12-05T07:40:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T07:40:58,208 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#A#compaction#188 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:58,208 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/4cbbad0771cb4a4e8a918cd1e60628df is 50, key is test_row_0/A:col10/1733384456158/Put/seqid=0 2024-12-05T07:40:58,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742055_1231 (size=12139) 2024-12-05T07:40:58,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742056_1232 (size=12139) 2024-12-05T07:40:58,220 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/a0a184ba5b71452b92f9e3b908b5fffc as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/a0a184ba5b71452b92f9e3b908b5fffc 2024-12-05T07:40:58,226 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/B of 063dfe74bcf23acd38effeda60a67fa8 into a0a184ba5b71452b92f9e3b908b5fffc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:58,226 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:40:58,226 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/B, priority=12, startTime=1733384458191; duration=0sec 2024-12-05T07:40:58,226 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:40:58,226 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:B 2024-12-05T07:40:58,226 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:40:58,227 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:40:58,227 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/C is initiating minor compaction (all files) 2024-12-05T07:40:58,228 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/C in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:58,228 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/80cf0af12d7e4f70ac16b0c2cd5d8e35, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/27c40958add64b4e9aae431d8bad6c8a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/dc06700c59e54ffba9b8760d88b62e3e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/2f372829774f4da69bb3b2bf5e606772] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=44.6 K 2024-12-05T07:40:58,228 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 80cf0af12d7e4f70ac16b0c2cd5d8e35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733384453884 2024-12-05T07:40:58,228 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 27c40958add64b4e9aae431d8bad6c8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733384453905 2024-12-05T07:40:58,229 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting dc06700c59e54ffba9b8760d88b62e3e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733384454542 2024-12-05T07:40:58,229 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f372829774f4da69bb3b2bf5e606772, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733384456144 2024-12-05T07:40:58,239 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#C#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:40:58,239 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/5ba9eca2e093425bab752dc507b2b19c is 50, key is test_row_0/C:col10/1733384456158/Put/seqid=0 2024-12-05T07:40:58,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742057_1233 (size=12139) 2024-12-05T07:40:58,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T07:40:58,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:58,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-05T07:40:58,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/e65523871d8245f3a7cd7a70d47ed6dc is 50, key is test_row_1/A:col10/1733384457321/Put/seqid=0 2024-12-05T07:40:58,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742058_1234 (size=9657) 2024-12-05T07:40:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T07:40:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:58,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:40:58,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384518529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:58,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:58,620 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/4cbbad0771cb4a4e8a918cd1e60628df as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/4cbbad0771cb4a4e8a918cd1e60628df 2024-12-05T07:40:58,625 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/A of 063dfe74bcf23acd38effeda60a67fa8 into 4cbbad0771cb4a4e8a918cd1e60628df(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:58,625 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:40:58,625 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/A, priority=12, startTime=1733384458190; duration=0sec 2024-12-05T07:40:58,625 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:58,625 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:A 2024-12-05T07:40:58,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384518636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:58,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:58,648 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/5ba9eca2e093425bab752dc507b2b19c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/5ba9eca2e093425bab752dc507b2b19c 2024-12-05T07:40:58,653 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/C of 063dfe74bcf23acd38effeda60a67fa8 into 5ba9eca2e093425bab752dc507b2b19c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:40:58,653 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:40:58,653 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/C, priority=12, startTime=1733384458191; duration=0sec 2024-12-05T07:40:58,653 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:40:58,653 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:C 2024-12-05T07:40:58,660 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/e65523871d8245f3a7cd7a70d47ed6dc 2024-12-05T07:40:58,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/89b7bb68b8184d84b618e6b6f5918132 is 50, key is test_row_1/B:col10/1733384457321/Put/seqid=0 2024-12-05T07:40:58,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742059_1235 (size=9657) 2024-12-05T07:40:58,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:58,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384518716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384518716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:58,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384518716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:58,718 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:58,719 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:58,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:58,719 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:58,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T07:40:58,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384518736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:58,737 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:40:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:58,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384518845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:58,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:59,072 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/89b7bb68b8184d84b618e6b6f5918132 2024-12-05T07:40:59,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/50c9321023a94aed85835e351d192f92 is 50, key is test_row_1/C:col10/1733384457321/Put/seqid=0 2024-12-05T07:40:59,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742060_1236 (size=9657) 2024-12-05T07:40:59,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384519156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T07:40:59,489 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/50c9321023a94aed85835e351d192f92 2024-12-05T07:40:59,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/e65523871d8245f3a7cd7a70d47ed6dc as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e65523871d8245f3a7cd7a70d47ed6dc 2024-12-05T07:40:59,500 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e65523871d8245f3a7cd7a70d47ed6dc, entries=100, sequenceid=86, filesize=9.4 K 2024-12-05T07:40:59,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/89b7bb68b8184d84b618e6b6f5918132 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/89b7bb68b8184d84b618e6b6f5918132 2024-12-05T07:40:59,505 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/89b7bb68b8184d84b618e6b6f5918132, entries=100, sequenceid=86, filesize=9.4 K 2024-12-05T07:40:59,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/50c9321023a94aed85835e351d192f92 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/50c9321023a94aed85835e351d192f92 2024-12-05T07:40:59,510 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/50c9321023a94aed85835e351d192f92, entries=100, sequenceid=86, filesize=9.4 K 2024-12-05T07:40:59,511 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 063dfe74bcf23acd38effeda60a67fa8 in 1259ms, sequenceid=86, compaction requested=false 2024-12-05T07:40:59,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:40:59,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:40:59,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-05T07:40:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-05T07:40:59,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-05T07:40:59,515 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-12-05T07:40:59,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.4190 sec 2024-12-05T07:40:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:40:59,668 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:40:59,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-12-05T07:40:59,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/e038fe8b67d2480c80aca2e63fc95fcf is 50, key is test_row_0/A:col10/1733384459667/Put/seqid=0 2024-12-05T07:40:59,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742061_1237 (size=12001) 2024-12-05T07:40:59,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384519676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:59,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384519786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:40:59,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:40:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384519995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:40:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:40:59,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:00,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/e038fe8b67d2480c80aca2e63fc95fcf 2024-12-05T07:41:00,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/6326d5eeff9b444ba88390f3705b5da5 is 50, key is test_row_0/B:col10/1733384459667/Put/seqid=0 2024-12-05T07:41:00,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742062_1238 (size=12001) 2024-12-05T07:41:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T07:41:00,246 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:00,246 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-05T07:41:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T07:41:00,247 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:00,248 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:00,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:00,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384520306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T07:41:00,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T07:41:00,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:00,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:00,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:00,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:00,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:00,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/6326d5eeff9b444ba88390f3705b5da5 2024-12-05T07:41:00,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/d7c9382cc8fe4c1ca9a5f8b9c22e3034 is 50, key is test_row_0/C:col10/1733384459667/Put/seqid=0 2024-12-05T07:41:00,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742063_1239 (size=12001) 2024-12-05T07:41:00,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/d7c9382cc8fe4c1ca9a5f8b9c22e3034 2024-12-05T07:41:00,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/e038fe8b67d2480c80aca2e63fc95fcf as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e038fe8b67d2480c80aca2e63fc95fcf 2024-12-05T07:41:00,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e038fe8b67d2480c80aca2e63fc95fcf, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T07:41:00,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/6326d5eeff9b444ba88390f3705b5da5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/6326d5eeff9b444ba88390f3705b5da5 2024-12-05T07:41:00,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/6326d5eeff9b444ba88390f3705b5da5, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T07:41:00,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/d7c9382cc8fe4c1ca9a5f8b9c22e3034 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/d7c9382cc8fe4c1ca9a5f8b9c22e3034 2024-12-05T07:41:00,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/d7c9382cc8fe4c1ca9a5f8b9c22e3034, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T07:41:00,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 063dfe74bcf23acd38effeda60a67fa8 in 863ms, sequenceid=118, compaction requested=true 2024-12-05T07:41:00,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:00,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:00,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:00,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:00,531 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:00,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:00,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:00,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:00,532 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:00,532 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:00,532 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:00,532 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/A is initiating minor compaction (all files) 2024-12-05T07:41:00,532 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/B is initiating minor compaction (all files) 2024-12-05T07:41:00,533 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/A in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:00,533 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/B in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:00,533 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/a0a184ba5b71452b92f9e3b908b5fffc, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/89b7bb68b8184d84b618e6b6f5918132, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/6326d5eeff9b444ba88390f3705b5da5] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=33.0 K 2024-12-05T07:41:00,533 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/4cbbad0771cb4a4e8a918cd1e60628df, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e65523871d8245f3a7cd7a70d47ed6dc, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e038fe8b67d2480c80aca2e63fc95fcf] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=33.0 K 2024-12-05T07:41:00,534 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cbbad0771cb4a4e8a918cd1e60628df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733384456144 2024-12-05T07:41:00,534 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a0a184ba5b71452b92f9e3b908b5fffc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733384456144 2024-12-05T07:41:00,535 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 89b7bb68b8184d84b618e6b6f5918132, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733384457321 2024-12-05T07:41:00,535 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e65523871d8245f3a7cd7a70d47ed6dc, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733384457321 2024-12-05T07:41:00,535 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e038fe8b67d2480c80aca2e63fc95fcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733384458516 2024-12-05T07:41:00,535 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6326d5eeff9b444ba88390f3705b5da5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733384458516 2024-12-05T07:41:00,542 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#A#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:00,542 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/5fcf6b0cc00e49e999f4b0fe565cbb87 is 50, key is test_row_0/A:col10/1733384459667/Put/seqid=0 2024-12-05T07:41:00,546 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#B#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:00,547 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/a80addb5af6c403185200192b96218b5 is 50, key is test_row_0/B:col10/1733384459667/Put/seqid=0 2024-12-05T07:41:00,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742064_1240 (size=12241) 2024-12-05T07:41:00,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T07:41:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:00,553 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-12-05T07:41:00,558 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/5fcf6b0cc00e49e999f4b0fe565cbb87 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/5fcf6b0cc00e49e999f4b0fe565cbb87 2024-12-05T07:41:00,563 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/A of 063dfe74bcf23acd38effeda60a67fa8 into 5fcf6b0cc00e49e999f4b0fe565cbb87(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:00,563 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:00,563 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/A, priority=13, startTime=1733384460531; duration=0sec 2024-12-05T07:41:00,563 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:00,563 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:A 2024-12-05T07:41:00,563 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:00,564 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:00,564 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/C is initiating minor compaction (all files) 2024-12-05T07:41:00,564 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/C in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:00,565 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/5ba9eca2e093425bab752dc507b2b19c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/50c9321023a94aed85835e351d192f92, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/d7c9382cc8fe4c1ca9a5f8b9c22e3034] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=33.0 K 2024-12-05T07:41:00,565 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ba9eca2e093425bab752dc507b2b19c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733384456144 2024-12-05T07:41:00,565 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c9321023a94aed85835e351d192f92, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733384457321 2024-12-05T07:41:00,565 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7c9382cc8fe4c1ca9a5f8b9c22e3034, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733384458516 2024-12-05T07:41:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T07:41:00,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742065_1241 (size=12241) 2024-12-05T07:41:00,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/034c7d0eab484593b06282f581454f07 is 50, key is test_row_1/A:col10/1733384459674/Put/seqid=0 2024-12-05T07:41:00,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742066_1242 (size=9657) 2024-12-05T07:41:00,576 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/a80addb5af6c403185200192b96218b5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/a80addb5af6c403185200192b96218b5 2024-12-05T07:41:00,577 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#C#compaction#200 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:00,577 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/d35d07afd74244e68b488ffeddf36f14 is 50, key is test_row_0/C:col10/1733384459667/Put/seqid=0 2024-12-05T07:41:00,577 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/034c7d0eab484593b06282f581454f07 2024-12-05T07:41:00,582 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/B of 063dfe74bcf23acd38effeda60a67fa8 into a80addb5af6c403185200192b96218b5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:00,583 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:00,583 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/B, priority=13, startTime=1733384460531; duration=0sec 2024-12-05T07:41:00,583 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:00,583 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:B 2024-12-05T07:41:00,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742067_1243 (size=12241) 2024-12-05T07:41:00,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/19751b6860914bb6adb743340b6d551d is 50, key is test_row_1/B:col10/1733384459674/Put/seqid=0 2024-12-05T07:41:00,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742068_1244 (size=9657) 2024-12-05T07:41:00,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T07:41:00,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384520891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:00,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:00,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:00,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384520995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:00,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:01,008 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/19751b6860914bb6adb743340b6d551d 2024-12-05T07:41:01,013 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/d35d07afd74244e68b488ffeddf36f14 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/d35d07afd74244e68b488ffeddf36f14 2024-12-05T07:41:01,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/7323abd34b7241d3b8d801b3cb7106dd is 50, key is test_row_1/C:col10/1733384459674/Put/seqid=0 2024-12-05T07:41:01,022 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/C of 063dfe74bcf23acd38effeda60a67fa8 into d35d07afd74244e68b488ffeddf36f14(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:01,022 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:01,022 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/C, priority=13, startTime=1733384460532; duration=0sec 2024-12-05T07:41:01,022 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:01,022 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:C 2024-12-05T07:41:01,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742069_1245 (size=9657) 2024-12-05T07:41:01,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384521205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T07:41:01,429 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/7323abd34b7241d3b8d801b3cb7106dd 2024-12-05T07:41:01,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/034c7d0eab484593b06282f581454f07 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/034c7d0eab484593b06282f581454f07 2024-12-05T07:41:01,439 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/034c7d0eab484593b06282f581454f07, entries=100, sequenceid=124, filesize=9.4 K 2024-12-05T07:41:01,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/19751b6860914bb6adb743340b6d551d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/19751b6860914bb6adb743340b6d551d 2024-12-05T07:41:01,445 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/19751b6860914bb6adb743340b6d551d, entries=100, sequenceid=124, filesize=9.4 K 2024-12-05T07:41:01,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/7323abd34b7241d3b8d801b3cb7106dd as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7323abd34b7241d3b8d801b3cb7106dd 2024-12-05T07:41:01,452 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7323abd34b7241d3b8d801b3cb7106dd, entries=100, sequenceid=124, filesize=9.4 K 2024-12-05T07:41:01,453 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 063dfe74bcf23acd38effeda60a67fa8 in 900ms, sequenceid=124, compaction requested=false 2024-12-05T07:41:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:01,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-05T07:41:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-05T07:41:01,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-05T07:41:01,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2090 sec 2024-12-05T07:41:01,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.2140 sec 2024-12-05T07:41:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:01,519 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:01,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-12-05T07:41:01,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/9b3f4cd1d30a451e9ef707d82771242e is 50, key is test_row_0/A:col10/1733384461517/Put/seqid=0 2024-12-05T07:41:01,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742070_1246 (size=14541) 2024-12-05T07:41:01,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/9b3f4cd1d30a451e9ef707d82771242e 2024-12-05T07:41:01,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384521531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:01,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:01,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:01,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:01,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/f2f30ff0f0604bff9e03daacf9f967a3 is 50, key is test_row_0/B:col10/1733384461517/Put/seqid=0 2024-12-05T07:41:01,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742071_1247 (size=12151) 2024-12-05T07:41:01,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/f2f30ff0f0604bff9e03daacf9f967a3 2024-12-05T07:41:01,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/7addcb64eb0d4d14beb84ee975fe4997 is 50, key is test_row_0/C:col10/1733384461517/Put/seqid=0 2024-12-05T07:41:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742072_1248 (size=12151) 2024-12-05T07:41:01,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/7addcb64eb0d4d14beb84ee975fe4997 2024-12-05T07:41:01,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/9b3f4cd1d30a451e9ef707d82771242e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9b3f4cd1d30a451e9ef707d82771242e 2024-12-05T07:41:01,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9b3f4cd1d30a451e9ef707d82771242e, entries=200, sequenceid=158, filesize=14.2 K 2024-12-05T07:41:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/f2f30ff0f0604bff9e03daacf9f967a3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f2f30ff0f0604bff9e03daacf9f967a3 2024-12-05T07:41:01,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f2f30ff0f0604bff9e03daacf9f967a3, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T07:41:01,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/7addcb64eb0d4d14beb84ee975fe4997 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7addcb64eb0d4d14beb84ee975fe4997 2024-12-05T07:41:01,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7addcb64eb0d4d14beb84ee975fe4997, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T07:41:01,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 063dfe74bcf23acd38effeda60a67fa8 in 102ms, sequenceid=158, compaction requested=true 2024-12-05T07:41:01,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:01,622 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:01,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:01,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:01,622 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:01,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:01,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:01,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:01,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:01,624 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:01,624 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:01,625 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/B is initiating minor compaction (all files) 2024-12-05T07:41:01,625 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/A is initiating minor compaction (all files) 2024-12-05T07:41:01,625 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/B in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:01,625 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/A in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:01,625 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/a80addb5af6c403185200192b96218b5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/19751b6860914bb6adb743340b6d551d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f2f30ff0f0604bff9e03daacf9f967a3] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=33.3 K 2024-12-05T07:41:01,625 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/5fcf6b0cc00e49e999f4b0fe565cbb87, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/034c7d0eab484593b06282f581454f07, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9b3f4cd1d30a451e9ef707d82771242e] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=35.6 K 2024-12-05T07:41:01,625 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fcf6b0cc00e49e999f4b0fe565cbb87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733384458516 2024-12-05T07:41:01,625 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a80addb5af6c403185200192b96218b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733384458516 2024-12-05T07:41:01,625 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 19751b6860914bb6adb743340b6d551d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733384459672 2024-12-05T07:41:01,625 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 034c7d0eab484593b06282f581454f07, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733384459672 2024-12-05T07:41:01,626 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f30ff0f0604bff9e03daacf9f967a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733384460880 2024-12-05T07:41:01,626 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b3f4cd1d30a451e9ef707d82771242e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733384460865 2024-12-05T07:41:01,651 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#A#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:01,652 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/9f2c31a86108480092cb6affadf25b94 is 50, key is test_row_0/A:col10/1733384461517/Put/seqid=0 2024-12-05T07:41:01,660 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:01,661 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/e690f87850d743f8846ed2b0d0cf6c0f is 50, key is test_row_0/B:col10/1733384461517/Put/seqid=0 2024-12-05T07:41:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:01,670 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:01,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:41:01,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742073_1249 (size=12493) 2024-12-05T07:41:01,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/509ef611ad5f41ff921f40d68ab2e815 is 50, key is test_row_0/A:col10/1733384461666/Put/seqid=0 2024-12-05T07:41:01,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742074_1250 (size=12493) 2024-12-05T07:41:01,705 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/e690f87850d743f8846ed2b0d0cf6c0f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/e690f87850d743f8846ed2b0d0cf6c0f 2024-12-05T07:41:01,713 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/B of 063dfe74bcf23acd38effeda60a67fa8 into e690f87850d743f8846ed2b0d0cf6c0f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:01,713 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:01,713 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/B, priority=13, startTime=1733384461622; duration=0sec 2024-12-05T07:41:01,713 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:01,713 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:B 2024-12-05T07:41:01,713 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:01,715 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:01,715 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/C is initiating minor compaction (all files) 2024-12-05T07:41:01,715 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/C in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:01,715 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/d35d07afd74244e68b488ffeddf36f14, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7323abd34b7241d3b8d801b3cb7106dd, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7addcb64eb0d4d14beb84ee975fe4997] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=33.3 K 2024-12-05T07:41:01,715 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting d35d07afd74244e68b488ffeddf36f14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733384458516 2024-12-05T07:41:01,716 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7323abd34b7241d3b8d801b3cb7106dd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1733384459672 2024-12-05T07:41:01,716 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7addcb64eb0d4d14beb84ee975fe4997, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733384460880 2024-12-05T07:41:01,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742075_1251 (size=14541) 2024-12-05T07:41:01,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/509ef611ad5f41ff921f40d68ab2e815 2024-12-05T07:41:01,733 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#C#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:01,734 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/43332a71c24646e0a52aeb6388fad215 is 50, key is test_row_0/C:col10/1733384461517/Put/seqid=0 2024-12-05T07:41:01,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/f042ef8f9db641eb9f9091f6cbcd5b0b is 50, key is test_row_0/B:col10/1733384461666/Put/seqid=0 2024-12-05T07:41:01,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742076_1252 (size=12493) 2024-12-05T07:41:01,775 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/43332a71c24646e0a52aeb6388fad215 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/43332a71c24646e0a52aeb6388fad215 2024-12-05T07:41:01,782 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/C of 063dfe74bcf23acd38effeda60a67fa8 into 43332a71c24646e0a52aeb6388fad215(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:01,782 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:01,782 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/C, priority=13, startTime=1733384461623; duration=0sec 2024-12-05T07:41:01,782 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:01,782 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:C 2024-12-05T07:41:01,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742077_1253 (size=12151) 2024-12-05T07:41:01,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/f042ef8f9db641eb9f9091f6cbcd5b0b 2024-12-05T07:41:01,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/ba6065ae3ded4f1ca6cb3e7af85db098 is 50, key is test_row_0/C:col10/1733384461666/Put/seqid=0 2024-12-05T07:41:01,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384521804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:01,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:01,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742078_1254 (size=12151) 2024-12-05T07:41:01,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384521916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:02,081 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/9f2c31a86108480092cb6affadf25b94 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9f2c31a86108480092cb6affadf25b94 2024-12-05T07:41:02,086 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/A of 063dfe74bcf23acd38effeda60a67fa8 into 9f2c31a86108480092cb6affadf25b94(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:02,086 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:02,086 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/A, priority=13, startTime=1733384461622; duration=0sec 2024-12-05T07:41:02,086 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:02,086 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:A 2024-12-05T07:41:02,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384522125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:02,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/ba6065ae3ded4f1ca6cb3e7af85db098 2024-12-05T07:41:02,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/509ef611ad5f41ff921f40d68ab2e815 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/509ef611ad5f41ff921f40d68ab2e815 2024-12-05T07:41:02,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/509ef611ad5f41ff921f40d68ab2e815, entries=200, sequenceid=169, filesize=14.2 K 2024-12-05T07:41:02,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/f042ef8f9db641eb9f9091f6cbcd5b0b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f042ef8f9db641eb9f9091f6cbcd5b0b 2024-12-05T07:41:02,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f042ef8f9db641eb9f9091f6cbcd5b0b, entries=150, sequenceid=169, filesize=11.9 K 2024-12-05T07:41:02,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/ba6065ae3ded4f1ca6cb3e7af85db098 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/ba6065ae3ded4f1ca6cb3e7af85db098 2024-12-05T07:41:02,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/ba6065ae3ded4f1ca6cb3e7af85db098, entries=150, sequenceid=169, filesize=11.9 K 2024-12-05T07:41:02,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 063dfe74bcf23acd38effeda60a67fa8 in 601ms, sequenceid=169, compaction requested=false 2024-12-05T07:41:02,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T07:41:02,396 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:02,396 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-05T07:41:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T07:41:02,399 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:02,400 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:02,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:02,438 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:02,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:02,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T07:41:02,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/77738a2a9d81410a9584186106c4736d is 50, key is test_row_0/A:col10/1733384461801/Put/seqid=0 2024-12-05T07:41:02,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742079_1255 (size=12151) 2024-12-05T07:41:02,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/77738a2a9d81410a9584186106c4736d 2024-12-05T07:41:02,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/f1ab31be2ab443288d518773ace387f9 is 50, key is test_row_0/B:col10/1733384461801/Put/seqid=0 2024-12-05T07:41:02,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384522476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:02,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742080_1256 (size=12151) 2024-12-05T07:41:02,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T07:41:02,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T07:41:02,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:02,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:02,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:02,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:02,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:02,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384522585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:02,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T07:41:02,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:02,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:02,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:02,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:02,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:02,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T07:41:02,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384522746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:02,756 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:02,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384522756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:02,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384522756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:02,758 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:02,759 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:02,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384522766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:02,768 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:02,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:02,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384522796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:02,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:02,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:02,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T07:41:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:02,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:02,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/f1ab31be2ab443288d518773ace387f9 2024-12-05T07:41:02,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/95613bf1f01c4bae9a09a0473e1e6525 is 50, key is test_row_0/C:col10/1733384461801/Put/seqid=0 2024-12-05T07:41:02,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742081_1257 (size=12151) 2024-12-05T07:41:03,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T07:41:03,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:03,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:03,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:03,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:03,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T07:41:03,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384523106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:03,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T07:41:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:03,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:03,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/95613bf1f01c4bae9a09a0473e1e6525 2024-12-05T07:41:03,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/77738a2a9d81410a9584186106c4736d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/77738a2a9d81410a9584186106c4736d 2024-12-05T07:41:03,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T07:41:03,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:03,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:03,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:03,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:03,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:03,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/77738a2a9d81410a9584186106c4736d, entries=150, sequenceid=198, filesize=11.9 K 2024-12-05T07:41:03,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/f1ab31be2ab443288d518773ace387f9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f1ab31be2ab443288d518773ace387f9 2024-12-05T07:41:03,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f1ab31be2ab443288d518773ace387f9, entries=150, sequenceid=198, filesize=11.9 K 2024-12-05T07:41:03,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/95613bf1f01c4bae9a09a0473e1e6525 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/95613bf1f01c4bae9a09a0473e1e6525 2024-12-05T07:41:03,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/95613bf1f01c4bae9a09a0473e1e6525, entries=150, sequenceid=198, filesize=11.9 K 2024-12-05T07:41:03,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 063dfe74bcf23acd38effeda60a67fa8 in 894ms, sequenceid=198, compaction requested=true 2024-12-05T07:41:03,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:03,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:03,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:03,334 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:03,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:03,334 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:03,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:03,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:03,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:03,335 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:03,335 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:03,335 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/B is initiating minor compaction (all files) 2024-12-05T07:41:03,335 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/A is initiating minor compaction (all files) 2024-12-05T07:41:03,335 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/A in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:03,335 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/B in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:03,335 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/e690f87850d743f8846ed2b0d0cf6c0f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f042ef8f9db641eb9f9091f6cbcd5b0b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f1ab31be2ab443288d518773ace387f9] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=35.9 K 2024-12-05T07:41:03,335 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9f2c31a86108480092cb6affadf25b94, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/509ef611ad5f41ff921f40d68ab2e815, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/77738a2a9d81410a9584186106c4736d] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=38.3 K 2024-12-05T07:41:03,336 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting e690f87850d743f8846ed2b0d0cf6c0f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733384460880 2024-12-05T07:41:03,336 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f2c31a86108480092cb6affadf25b94, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733384460880 2024-12-05T07:41:03,336 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 509ef611ad5f41ff921f40d68ab2e815, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733384461649 2024-12-05T07:41:03,336 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f042ef8f9db641eb9f9091f6cbcd5b0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733384461663 2024-12-05T07:41:03,336 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77738a2a9d81410a9584186106c4736d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384461796 2024-12-05T07:41:03,337 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ab31be2ab443288d518773ace387f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384461796 2024-12-05T07:41:03,347 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#B#compaction#215 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:03,347 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/883ff3f0d96346bc89bd85ef916c26d7 is 50, key is test_row_0/B:col10/1733384461801/Put/seqid=0 2024-12-05T07:41:03,350 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#A#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:03,350 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/94be4a7379074a6da87a73a29c6f2506 is 50, key is test_row_0/A:col10/1733384461801/Put/seqid=0 2024-12-05T07:41:03,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742082_1258 (size=12595) 2024-12-05T07:41:03,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742083_1259 (size=12595) 2024-12-05T07:41:03,378 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/883ff3f0d96346bc89bd85ef916c26d7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/883ff3f0d96346bc89bd85ef916c26d7 2024-12-05T07:41:03,378 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/94be4a7379074a6da87a73a29c6f2506 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/94be4a7379074a6da87a73a29c6f2506 2024-12-05T07:41:03,386 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/B of 063dfe74bcf23acd38effeda60a67fa8 into 883ff3f0d96346bc89bd85ef916c26d7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:03,386 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:03,386 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/B, priority=13, startTime=1733384463334; duration=0sec 2024-12-05T07:41:03,386 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:03,386 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:B 2024-12-05T07:41:03,386 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:03,388 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/A of 063dfe74bcf23acd38effeda60a67fa8 into 94be4a7379074a6da87a73a29c6f2506(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:03,388 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:03,388 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/A, priority=13, startTime=1733384463334; duration=0sec 2024-12-05T07:41:03,388 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:03,388 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/C is initiating minor compaction (all files) 2024-12-05T07:41:03,388 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:03,388 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:A 2024-12-05T07:41:03,388 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/C in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:03,388 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/43332a71c24646e0a52aeb6388fad215, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/ba6065ae3ded4f1ca6cb3e7af85db098, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/95613bf1f01c4bae9a09a0473e1e6525] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=35.9 K 2024-12-05T07:41:03,389 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 43332a71c24646e0a52aeb6388fad215, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733384460880 2024-12-05T07:41:03,390 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting ba6065ae3ded4f1ca6cb3e7af85db098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1733384461663 2024-12-05T07:41:03,390 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 95613bf1f01c4bae9a09a0473e1e6525, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384461796 2024-12-05T07:41:03,398 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#C#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:03,399 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/59a200249b1f4f7684afbf12e0f0da5f is 50, key is test_row_0/C:col10/1733384461801/Put/seqid=0 2024-12-05T07:41:03,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742084_1260 (size=12595) 2024-12-05T07:41:03,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T07:41:03,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:03,469 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-05T07:41:03,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/1630b053863d4d5cbaa80d192e92cdc5 is 50, key is test_row_0/A:col10/1733384462472/Put/seqid=0 2024-12-05T07:41:03,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742085_1261 (size=12151) 2024-12-05T07:41:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T07:41:03,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:03,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384523692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:03,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384523806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:03,846 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/59a200249b1f4f7684afbf12e0f0da5f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/59a200249b1f4f7684afbf12e0f0da5f 2024-12-05T07:41:03,850 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/C of 063dfe74bcf23acd38effeda60a67fa8 into 59a200249b1f4f7684afbf12e0f0da5f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:03,850 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:03,850 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/C, priority=13, startTime=1733384463334; duration=0sec 2024-12-05T07:41:03,850 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:03,850 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:C 2024-12-05T07:41:03,885 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/1630b053863d4d5cbaa80d192e92cdc5 2024-12-05T07:41:03,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/733877547e7b4b2690a14bdce02e8306 is 50, key is test_row_0/B:col10/1733384462472/Put/seqid=0 2024-12-05T07:41:03,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742086_1262 (size=12151) 2024-12-05T07:41:04,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384524016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:04,309 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/733877547e7b4b2690a14bdce02e8306 2024-12-05T07:41:04,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/34819125e37c4601add11b757c9dc63c is 50, key is test_row_0/C:col10/1733384462472/Put/seqid=0 2024-12-05T07:41:04,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384524326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:04,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742087_1263 (size=12151) 2024-12-05T07:41:04,352 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/34819125e37c4601add11b757c9dc63c 2024-12-05T07:41:04,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/1630b053863d4d5cbaa80d192e92cdc5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1630b053863d4d5cbaa80d192e92cdc5 2024-12-05T07:41:04,365 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1630b053863d4d5cbaa80d192e92cdc5, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T07:41:04,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/733877547e7b4b2690a14bdce02e8306 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/733877547e7b4b2690a14bdce02e8306 2024-12-05T07:41:04,380 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/733877547e7b4b2690a14bdce02e8306, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T07:41:04,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/34819125e37c4601add11b757c9dc63c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/34819125e37c4601add11b757c9dc63c 2024-12-05T07:41:04,389 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/34819125e37c4601add11b757c9dc63c, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T07:41:04,390 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 063dfe74bcf23acd38effeda60a67fa8 in 920ms, sequenceid=210, compaction requested=false 2024-12-05T07:41:04,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:04,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:04,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-05T07:41:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-05T07:41:04,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-05T07:41:04,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9910 sec 2024-12-05T07:41:04,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.9990 sec 2024-12-05T07:41:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T07:41:04,546 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:04,547 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-05T07:41:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T07:41:04,549 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:04,550 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:04,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T07:41:04,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T07:41:04,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:04,704 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T07:41:04,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/2c0b8c747a1d472e8a2f1cc93eeb3d71 is 50, key is test_row_0/A:col10/1733384463682/Put/seqid=0 2024-12-05T07:41:04,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742088_1264 (size=12151) 2024-12-05T07:41:04,754 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/2c0b8c747a1d472e8a2f1cc93eeb3d71 2024-12-05T07:41:04,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/e38e0e54ffd94237b4b339786acf02f2 is 50, key is test_row_0/B:col10/1733384463682/Put/seqid=0 2024-12-05T07:41:04,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742089_1265 (size=12151) 2024-12-05T07:41:04,801 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/e38e0e54ffd94237b4b339786acf02f2 2024-12-05T07:41:04,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/4ce2a43ceb7a4f0b86667eda217117f3 is 50, key is test_row_0/C:col10/1733384463682/Put/seqid=0 2024-12-05T07:41:04,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:04,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742090_1266 (size=12151) 2024-12-05T07:41:04,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T07:41:04,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:04,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384524872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:04,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:04,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:04,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:04,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384524986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T07:41:05,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384525196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:05,253 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/4ce2a43ceb7a4f0b86667eda217117f3 2024-12-05T07:41:05,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/2c0b8c747a1d472e8a2f1cc93eeb3d71 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/2c0b8c747a1d472e8a2f1cc93eeb3d71 2024-12-05T07:41:05,262 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/2c0b8c747a1d472e8a2f1cc93eeb3d71, entries=150, sequenceid=237, filesize=11.9 K 2024-12-05T07:41:05,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/e38e0e54ffd94237b4b339786acf02f2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/e38e0e54ffd94237b4b339786acf02f2 2024-12-05T07:41:05,269 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/e38e0e54ffd94237b4b339786acf02f2, entries=150, sequenceid=237, filesize=11.9 K 2024-12-05T07:41:05,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/4ce2a43ceb7a4f0b86667eda217117f3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4ce2a43ceb7a4f0b86667eda217117f3 2024-12-05T07:41:05,274 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4ce2a43ceb7a4f0b86667eda217117f3, entries=150, sequenceid=237, filesize=11.9 K 2024-12-05T07:41:05,275 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 063dfe74bcf23acd38effeda60a67fa8 in 572ms, sequenceid=237, compaction requested=true 2024-12-05T07:41:05,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:05,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:05,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-05T07:41:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-05T07:41:05,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-05T07:41:05,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 726 msec 2024-12-05T07:41:05,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 732 msec 2024-12-05T07:41:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:05,507 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:05,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:41:05,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/1510782636684e7dba420522182381ed is 50, key is test_row_0/A:col10/1733384465506/Put/seqid=0 2024-12-05T07:41:05,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742091_1267 (size=14541) 2024-12-05T07:41:05,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384525609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T07:41:05,686 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:05,687 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-05T07:41:05,688 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:05,689 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:05,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T07:41:05,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384525716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T07:41:05,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T07:41:05,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:05,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:05,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:05,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:05,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:05,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384525926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:05,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/1510782636684e7dba420522182381ed 2024-12-05T07:41:05,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/7769c7efe81743bbbe2ea1d91d5e6ba6 is 50, key is test_row_0/B:col10/1733384465506/Put/seqid=0 2024-12-05T07:41:05,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742092_1268 (size=12151) 2024-12-05T07:41:05,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T07:41:05,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:05,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:05,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:05,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:05,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T07:41:06,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T07:41:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:06,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384526236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:06,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T07:41:06,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:06,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T07:41:06,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/7769c7efe81743bbbe2ea1d91d5e6ba6 2024-12-05T07:41:06,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/9ed509feba95467eb93802607dfa8d72 is 50, key is test_row_0/C:col10/1733384465506/Put/seqid=0 2024-12-05T07:41:06,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742093_1269 (size=12151) 2024-12-05T07:41:06,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T07:41:06,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:06,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T07:41:06,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:06,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384526746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:06,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T07:41:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:06,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:06,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/9ed509feba95467eb93802607dfa8d72 2024-12-05T07:41:06,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/1510782636684e7dba420522182381ed as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1510782636684e7dba420522182381ed 2024-12-05T07:41:06,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1510782636684e7dba420522182381ed, entries=200, sequenceid=248, filesize=14.2 K 2024-12-05T07:41:06,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/7769c7efe81743bbbe2ea1d91d5e6ba6 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/7769c7efe81743bbbe2ea1d91d5e6ba6 2024-12-05T07:41:06,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/7769c7efe81743bbbe2ea1d91d5e6ba6, entries=150, sequenceid=248, filesize=11.9 K 2024-12-05T07:41:06,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/9ed509feba95467eb93802607dfa8d72 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/9ed509feba95467eb93802607dfa8d72 2024-12-05T07:41:06,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/9ed509feba95467eb93802607dfa8d72, entries=150, sequenceid=248, filesize=11.9 K 2024-12-05T07:41:06,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 063dfe74bcf23acd38effeda60a67fa8 in 1275ms, sequenceid=248, compaction requested=true 2024-12-05T07:41:06,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:06,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:06,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:06,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:06,783 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:41:06,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:06,783 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:41:06,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:06,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:06,784 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:41:06,784 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:41:06,784 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/B is initiating minor compaction (all files) 2024-12-05T07:41:06,784 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/A is initiating minor compaction (all files) 2024-12-05T07:41:06,784 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/A in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,784 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/B in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,784 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/883ff3f0d96346bc89bd85ef916c26d7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/733877547e7b4b2690a14bdce02e8306, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/e38e0e54ffd94237b4b339786acf02f2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/7769c7efe81743bbbe2ea1d91d5e6ba6] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=47.9 K 2024-12-05T07:41:06,784 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/94be4a7379074a6da87a73a29c6f2506, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1630b053863d4d5cbaa80d192e92cdc5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/2c0b8c747a1d472e8a2f1cc93eeb3d71, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1510782636684e7dba420522182381ed] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=50.2 K 2024-12-05T07:41:06,785 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94be4a7379074a6da87a73a29c6f2506, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384461796 2024-12-05T07:41:06,785 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 883ff3f0d96346bc89bd85ef916c26d7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384461796 2024-12-05T07:41:06,785 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1630b053863d4d5cbaa80d192e92cdc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733384462456 2024-12-05T07:41:06,785 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 733877547e7b4b2690a14bdce02e8306, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733384462456 2024-12-05T07:41:06,785 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c0b8c747a1d472e8a2f1cc93eeb3d71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733384463678 2024-12-05T07:41:06,785 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting e38e0e54ffd94237b4b339786acf02f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733384463678 2024-12-05T07:41:06,785 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1510782636684e7dba420522182381ed, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733384464857 2024-12-05T07:41:06,786 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7769c7efe81743bbbe2ea1d91d5e6ba6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733384464857 2024-12-05T07:41:06,792 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#B#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:06,793 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/d287b92a138948fbb9d6a98d96e6b2da is 50, key is test_row_0/B:col10/1733384465506/Put/seqid=0 2024-12-05T07:41:06,796 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:06,796 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/05d2e4c11e22482ca680eee4d126b538 is 50, key is test_row_0/A:col10/1733384465506/Put/seqid=0 2024-12-05T07:41:06,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742094_1270 (size=12731) 2024-12-05T07:41:06,821 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/d287b92a138948fbb9d6a98d96e6b2da as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/d287b92a138948fbb9d6a98d96e6b2da 2024-12-05T07:41:06,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742095_1271 (size=12731) 2024-12-05T07:41:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T07:41:06,828 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/B of 063dfe74bcf23acd38effeda60a67fa8 into d287b92a138948fbb9d6a98d96e6b2da(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:06,828 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:06,828 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/B, priority=12, startTime=1733384466783; duration=0sec 2024-12-05T07:41:06,828 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:06,828 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:B 2024-12-05T07:41:06,828 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:41:06,830 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:41:06,830 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/C is initiating minor compaction (all files) 2024-12-05T07:41:06,831 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/C in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,831 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/59a200249b1f4f7684afbf12e0f0da5f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/34819125e37c4601add11b757c9dc63c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4ce2a43ceb7a4f0b86667eda217117f3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/9ed509feba95467eb93802607dfa8d72] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=47.9 K 2024-12-05T07:41:06,831 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/05d2e4c11e22482ca680eee4d126b538 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/05d2e4c11e22482ca680eee4d126b538 2024-12-05T07:41:06,831 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a200249b1f4f7684afbf12e0f0da5f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384461796 2024-12-05T07:41:06,832 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 34819125e37c4601add11b757c9dc63c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733384462456 2024-12-05T07:41:06,832 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce2a43ceb7a4f0b86667eda217117f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733384463678 2024-12-05T07:41:06,834 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ed509feba95467eb93802607dfa8d72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733384464857 2024-12-05T07:41:06,837 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/A of 063dfe74bcf23acd38effeda60a67fa8 into 05d2e4c11e22482ca680eee4d126b538(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:06,837 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:06,837 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/A, priority=12, startTime=1733384466783; duration=0sec 2024-12-05T07:41:06,837 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:06,837 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:A 2024-12-05T07:41:06,853 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#C#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:06,853 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/658f06dc2e11406e92d832d1f214cdd7 is 50, key is test_row_0/C:col10/1733384465506/Put/seqid=0 2024-12-05T07:41:06,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742096_1272 (size=12731) 2024-12-05T07:41:06,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T07:41:06,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:06,919 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T07:41:06,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/505b07da413d4e658fe194a20604e60b is 50, key is test_row_0/A:col10/1733384465601/Put/seqid=0 2024-12-05T07:41:06,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742097_1273 (size=12301) 2024-12-05T07:41:07,284 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/658f06dc2e11406e92d832d1f214cdd7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/658f06dc2e11406e92d832d1f214cdd7 2024-12-05T07:41:07,299 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/C of 063dfe74bcf23acd38effeda60a67fa8 into 658f06dc2e11406e92d832d1f214cdd7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:07,299 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:07,299 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/C, priority=12, startTime=1733384466783; duration=0sec 2024-12-05T07:41:07,299 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:07,299 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:C 2024-12-05T07:41:07,350 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/505b07da413d4e658fe194a20604e60b 2024-12-05T07:41:07,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/39e9563fbc7a49f98bc475469fb07180 is 50, key is test_row_0/B:col10/1733384465601/Put/seqid=0 2024-12-05T07:41:07,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742098_1274 (size=12301) 2024-12-05T07:41:07,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:07,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:07,774 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/39e9563fbc7a49f98bc475469fb07180 2024-12-05T07:41:07,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/b0d6bc0dac304bf284bc631aa4d07a58 is 50, key is test_row_0/C:col10/1733384465601/Put/seqid=0 2024-12-05T07:41:07,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:07,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384527786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:07,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742099_1275 (size=12301) 2024-12-05T07:41:07,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T07:41:07,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384527896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:08,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384528105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:08,204 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/b0d6bc0dac304bf284bc631aa4d07a58 2024-12-05T07:41:08,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/505b07da413d4e658fe194a20604e60b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/505b07da413d4e658fe194a20604e60b 2024-12-05T07:41:08,212 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/505b07da413d4e658fe194a20604e60b, entries=150, sequenceid=275, filesize=12.0 K 2024-12-05T07:41:08,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/39e9563fbc7a49f98bc475469fb07180 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39e9563fbc7a49f98bc475469fb07180 2024-12-05T07:41:08,217 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39e9563fbc7a49f98bc475469fb07180, entries=150, sequenceid=275, filesize=12.0 K 2024-12-05T07:41:08,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/b0d6bc0dac304bf284bc631aa4d07a58 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/b0d6bc0dac304bf284bc631aa4d07a58 2024-12-05T07:41:08,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/b0d6bc0dac304bf284bc631aa4d07a58, entries=150, sequenceid=275, filesize=12.0 K 2024-12-05T07:41:08,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 063dfe74bcf23acd38effeda60a67fa8 in 1304ms, sequenceid=275, compaction requested=false 2024-12-05T07:41:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:08,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-05T07:41:08,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-05T07:41:08,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-05T07:41:08,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5360 sec 2024-12-05T07:41:08,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.5400 sec 2024-12-05T07:41:08,419 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:08,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T07:41:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:08,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/8a01f292661641d48abb5c43c5aaf79b is 50, key is test_row_0/A:col10/1733384468417/Put/seqid=0 2024-12-05T07:41:08,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742100_1276 (size=12301) 2024-12-05T07:41:08,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384528523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:08,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384528636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:08,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384528846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:08,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:08,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:08,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:08,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/8a01f292661641d48abb5c43c5aaf79b 2024-12-05T07:41:08,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/50d921f2719a4602a3274a90d1dfac10 is 50, key is test_row_0/B:col10/1733384468417/Put/seqid=0 2024-12-05T07:41:08,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742101_1277 (size=12301) 2024-12-05T07:41:09,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384529156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:09,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:09,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/50d921f2719a4602a3274a90d1dfac10 2024-12-05T07:41:09,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/4d645688561f43bc9e6f29d1fa525aaf is 50, key is test_row_0/C:col10/1733384468417/Put/seqid=0 2024-12-05T07:41:09,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742102_1278 (size=12301) 2024-12-05T07:41:09,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384529666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:09,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/4d645688561f43bc9e6f29d1fa525aaf 2024-12-05T07:41:09,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/8a01f292661641d48abb5c43c5aaf79b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/8a01f292661641d48abb5c43c5aaf79b 2024-12-05T07:41:09,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/8a01f292661641d48abb5c43c5aaf79b, entries=150, sequenceid=288, filesize=12.0 K 2024-12-05T07:41:09,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/50d921f2719a4602a3274a90d1dfac10 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/50d921f2719a4602a3274a90d1dfac10 2024-12-05T07:41:09,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/50d921f2719a4602a3274a90d1dfac10, entries=150, sequenceid=288, filesize=12.0 K 2024-12-05T07:41:09,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/4d645688561f43bc9e6f29d1fa525aaf as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4d645688561f43bc9e6f29d1fa525aaf 2024-12-05T07:41:09,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4d645688561f43bc9e6f29d1fa525aaf, entries=150, sequenceid=288, filesize=12.0 K 2024-12-05T07:41:09,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 063dfe74bcf23acd38effeda60a67fa8 in 1286ms, sequenceid=288, compaction requested=true 2024-12-05T07:41:09,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:09,705 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:09,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:09,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:09,706 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:09,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:09,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:09,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:09,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:09,707 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:09,707 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/A is initiating minor compaction (all files) 2024-12-05T07:41:09,707 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/A in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:09,707 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/05d2e4c11e22482ca680eee4d126b538, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/505b07da413d4e658fe194a20604e60b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/8a01f292661641d48abb5c43c5aaf79b] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=36.5 K 2024-12-05T07:41:09,707 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:09,707 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/B is initiating minor compaction (all files) 2024-12-05T07:41:09,707 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/B in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:09,708 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/d287b92a138948fbb9d6a98d96e6b2da, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39e9563fbc7a49f98bc475469fb07180, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/50d921f2719a4602a3274a90d1dfac10] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=36.5 K 2024-12-05T07:41:09,708 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05d2e4c11e22482ca680eee4d126b538, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733384464857 2024-12-05T07:41:09,708 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting d287b92a138948fbb9d6a98d96e6b2da, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733384464857 2024-12-05T07:41:09,708 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 505b07da413d4e658fe194a20604e60b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733384465595 2024-12-05T07:41:09,709 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 39e9563fbc7a49f98bc475469fb07180, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733384465595 2024-12-05T07:41:09,709 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a01f292661641d48abb5c43c5aaf79b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733384467760 2024-12-05T07:41:09,710 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 50d921f2719a4602a3274a90d1dfac10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733384467760 2024-12-05T07:41:09,720 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#B#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:09,720 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/00373c50b67d4c1aa301bf65f819c2eb is 50, key is test_row_0/B:col10/1733384468417/Put/seqid=0 2024-12-05T07:41:09,724 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:09,725 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/326483f4eb594740bdee0de7f4665828 is 50, key is test_row_0/A:col10/1733384468417/Put/seqid=0 2024-12-05T07:41:09,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742103_1279 (size=12983) 2024-12-05T07:41:09,737 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/00373c50b67d4c1aa301bf65f819c2eb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/00373c50b67d4c1aa301bf65f819c2eb 2024-12-05T07:41:09,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742104_1280 (size=12983) 2024-12-05T07:41:09,743 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/B of 063dfe74bcf23acd38effeda60a67fa8 into 00373c50b67d4c1aa301bf65f819c2eb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:09,743 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:09,743 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/B, priority=13, startTime=1733384469706; duration=0sec 2024-12-05T07:41:09,743 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:09,743 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:B 2024-12-05T07:41:09,743 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:09,745 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:09,745 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/C is initiating minor compaction (all files) 2024-12-05T07:41:09,745 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/C in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:09,745 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/658f06dc2e11406e92d832d1f214cdd7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/b0d6bc0dac304bf284bc631aa4d07a58, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4d645688561f43bc9e6f29d1fa525aaf] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=36.5 K 2024-12-05T07:41:09,746 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 658f06dc2e11406e92d832d1f214cdd7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733384464857 2024-12-05T07:41:09,747 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting b0d6bc0dac304bf284bc631aa4d07a58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733384465595 2024-12-05T07:41:09,747 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d645688561f43bc9e6f29d1fa525aaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733384467760 2024-12-05T07:41:09,749 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/326483f4eb594740bdee0de7f4665828 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/326483f4eb594740bdee0de7f4665828 2024-12-05T07:41:09,756 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/A of 063dfe74bcf23acd38effeda60a67fa8 into 326483f4eb594740bdee0de7f4665828(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:09,756 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:09,756 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/A, priority=13, startTime=1733384469705; duration=0sec 2024-12-05T07:41:09,756 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:09,756 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:A 2024-12-05T07:41:09,758 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#C#compaction#238 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:09,759 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/c9a4f18c3cfe4a3ea7dc26c1134e16be is 50, key is test_row_0/C:col10/1733384468417/Put/seqid=0 2024-12-05T07:41:09,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742105_1281 (size=12983) 2024-12-05T07:41:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T07:41:09,856 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:09,856 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-05T07:41:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T07:41:09,859 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:09,859 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:09,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T07:41:10,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T07:41:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:10,012 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T07:41:10,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/aedc7f084de4427a9e08dc442d40ddfb is 50, key is test_row_0/A:col10/1733384468516/Put/seqid=0 2024-12-05T07:41:10,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742106_1282 (size=12301) 2024-12-05T07:41:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T07:41:10,182 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/c9a4f18c3cfe4a3ea7dc26c1134e16be as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/c9a4f18c3cfe4a3ea7dc26c1134e16be 2024-12-05T07:41:10,186 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/C of 063dfe74bcf23acd38effeda60a67fa8 into c9a4f18c3cfe4a3ea7dc26c1134e16be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:10,186 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:10,186 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/C, priority=13, startTime=1733384469706; duration=0sec 2024-12-05T07:41:10,186 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:10,186 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:C 2024-12-05T07:41:10,466 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/aedc7f084de4427a9e08dc442d40ddfb 2024-12-05T07:41:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/1cbc0ab232334e72b947051b967accd9 is 50, key is test_row_0/B:col10/1733384468516/Put/seqid=0 2024-12-05T07:41:10,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742107_1283 (size=12301) 2024-12-05T07:41:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T07:41:10,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:10,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384530717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:10,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:10,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384530826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:10,882 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/1cbc0ab232334e72b947051b967accd9 2024-12-05T07:41:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/fcf5409df77b46acb68aed17d13eeaba is 50, key is test_row_0/C:col10/1733384468516/Put/seqid=0 2024-12-05T07:41:10,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742108_1284 (size=12301) 2024-12-05T07:41:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T07:41:11,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384531037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:11,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:11,299 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/fcf5409df77b46acb68aed17d13eeaba 2024-12-05T07:41:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/aedc7f084de4427a9e08dc442d40ddfb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/aedc7f084de4427a9e08dc442d40ddfb 2024-12-05T07:41:11,309 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/aedc7f084de4427a9e08dc442d40ddfb, entries=150, sequenceid=314, filesize=12.0 K 2024-12-05T07:41:11,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/1cbc0ab232334e72b947051b967accd9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/1cbc0ab232334e72b947051b967accd9 2024-12-05T07:41:11,313 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/1cbc0ab232334e72b947051b967accd9, entries=150, sequenceid=314, filesize=12.0 K 2024-12-05T07:41:11,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/fcf5409df77b46acb68aed17d13eeaba as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/fcf5409df77b46acb68aed17d13eeaba 2024-12-05T07:41:11,318 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/fcf5409df77b46acb68aed17d13eeaba, entries=150, sequenceid=314, filesize=12.0 K 2024-12-05T07:41:11,319 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 063dfe74bcf23acd38effeda60a67fa8 in 1308ms, sequenceid=314, compaction requested=false 2024-12-05T07:41:11,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:11,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:11,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-05T07:41:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-05T07:41:11,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-05T07:41:11,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4620 sec 2024-12-05T07:41:11,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4670 sec 2024-12-05T07:41:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:11,348 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:11,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T07:41:11,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/122a88ab73da4c38bb3a9cbec38e3d5b is 50, key is test_row_0/A:col10/1733384471347/Put/seqid=0 2024-12-05T07:41:11,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742109_1285 (size=12301) 2024-12-05T07:41:11,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384531436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:11,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:11,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384531546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:11,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:11,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/122a88ab73da4c38bb3a9cbec38e3d5b 2024-12-05T07:41:11,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384531755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:11,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:11,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/ba1ee0c0ea24466c90d78c231d8d8ed6 is 50, key is test_row_0/B:col10/1733384471347/Put/seqid=0 2024-12-05T07:41:11,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742110_1286 (size=12301) 2024-12-05T07:41:11,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/ba1ee0c0ea24466c90d78c231d8d8ed6 2024-12-05T07:41:11,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/6b2c7eee20c049ff8494791198c2764e is 50, key is test_row_0/C:col10/1733384471347/Put/seqid=0 2024-12-05T07:41:11,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742111_1287 (size=12301) 2024-12-05T07:41:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T07:41:12,006 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:12,006 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-05T07:41:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T07:41:12,010 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:12,010 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:12,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:12,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384532066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T07:41:12,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T07:41:12,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:12,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:12,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:12,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:12,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:12,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/6b2c7eee20c049ff8494791198c2764e 2024-12-05T07:41:12,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/122a88ab73da4c38bb3a9cbec38e3d5b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/122a88ab73da4c38bb3a9cbec38e3d5b 2024-12-05T07:41:12,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/122a88ab73da4c38bb3a9cbec38e3d5b, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T07:41:12,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/ba1ee0c0ea24466c90d78c231d8d8ed6 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ba1ee0c0ea24466c90d78c231d8d8ed6 2024-12-05T07:41:12,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ba1ee0c0ea24466c90d78c231d8d8ed6, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T07:41:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/6b2c7eee20c049ff8494791198c2764e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6b2c7eee20c049ff8494791198c2764e 2024-12-05T07:41:12,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6b2c7eee20c049ff8494791198c2764e, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T07:41:12,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 063dfe74bcf23acd38effeda60a67fa8 in 895ms, sequenceid=328, compaction requested=true 2024-12-05T07:41:12,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:12,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:12,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:12,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:12,244 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:12,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:12,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:12,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:12,244 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:12,245 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:12,245 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:12,245 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/B is initiating minor compaction (all files) 2024-12-05T07:41:12,245 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/A is initiating minor compaction (all files) 2024-12-05T07:41:12,245 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/A in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:12,245 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/B in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:12,245 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/326483f4eb594740bdee0de7f4665828, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/aedc7f084de4427a9e08dc442d40ddfb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/122a88ab73da4c38bb3a9cbec38e3d5b] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=36.7 K 2024-12-05T07:41:12,245 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/00373c50b67d4c1aa301bf65f819c2eb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/1cbc0ab232334e72b947051b967accd9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ba1ee0c0ea24466c90d78c231d8d8ed6] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=36.7 K 2024-12-05T07:41:12,246 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 00373c50b67d4c1aa301bf65f819c2eb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733384467760 2024-12-05T07:41:12,246 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 326483f4eb594740bdee0de7f4665828, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733384467760 2024-12-05T07:41:12,246 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting aedc7f084de4427a9e08dc442d40ddfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733384468498 2024-12-05T07:41:12,246 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cbc0ab232334e72b947051b967accd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733384468498 2024-12-05T07:41:12,246 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 122a88ab73da4c38bb3a9cbec38e3d5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733384470703 2024-12-05T07:41:12,246 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting ba1ee0c0ea24466c90d78c231d8d8ed6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733384470703 2024-12-05T07:41:12,253 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#B#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:12,253 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:12,254 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/84dc8192124a45838d24a699c0051e55 is 50, key is test_row_0/A:col10/1733384471347/Put/seqid=0 2024-12-05T07:41:12,254 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/3217b506f2014a00b843e75162b560b7 is 50, key is test_row_0/B:col10/1733384471347/Put/seqid=0 2024-12-05T07:41:12,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742113_1289 (size=13085) 2024-12-05T07:41:12,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742112_1288 (size=13085) 2024-12-05T07:41:12,263 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/84dc8192124a45838d24a699c0051e55 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/84dc8192124a45838d24a699c0051e55 2024-12-05T07:41:12,276 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/A of 063dfe74bcf23acd38effeda60a67fa8 into 84dc8192124a45838d24a699c0051e55(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:12,277 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:12,277 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/A, priority=13, startTime=1733384472244; duration=0sec 2024-12-05T07:41:12,277 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:12,277 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:A 2024-12-05T07:41:12,277 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:12,278 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:12,278 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/C is initiating minor compaction (all files) 2024-12-05T07:41:12,278 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/C in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:12,278 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/c9a4f18c3cfe4a3ea7dc26c1134e16be, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/fcf5409df77b46acb68aed17d13eeaba, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6b2c7eee20c049ff8494791198c2764e] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=36.7 K 2024-12-05T07:41:12,279 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9a4f18c3cfe4a3ea7dc26c1134e16be, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733384467760 2024-12-05T07:41:12,279 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf5409df77b46acb68aed17d13eeaba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733384468498 2024-12-05T07:41:12,279 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b2c7eee20c049ff8494791198c2764e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733384470703 2024-12-05T07:41:12,297 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#C#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:12,298 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/dd4cf6bd7e1f47b99cf0640fe0312a52 is 50, key is test_row_0/C:col10/1733384471347/Put/seqid=0 2024-12-05T07:41:12,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742114_1290 (size=13085) 2024-12-05T07:41:12,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T07:41:12,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:12,316 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T07:41:12,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/938b7cbf4cb94643baabf15744adb2b9 is 50, key is test_row_0/A:col10/1733384471435/Put/seqid=0 2024-12-05T07:41:12,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742115_1291 (size=12301) 2024-12-05T07:41:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T07:41:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:12,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. as already flushing 2024-12-05T07:41:12,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384532624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T07:41:12,665 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/3217b506f2014a00b843e75162b560b7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/3217b506f2014a00b843e75162b560b7 2024-12-05T07:41:12,671 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/B of 063dfe74bcf23acd38effeda60a67fa8 into 3217b506f2014a00b843e75162b560b7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:12,671 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:12,671 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/B, priority=13, startTime=1733384472244; duration=0sec 2024-12-05T07:41:12,671 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:12,671 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:B 2024-12-05T07:41:12,717 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/dd4cf6bd7e1f47b99cf0640fe0312a52 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/dd4cf6bd7e1f47b99cf0640fe0312a52 2024-12-05T07:41:12,724 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/C of 063dfe74bcf23acd38effeda60a67fa8 into dd4cf6bd7e1f47b99cf0640fe0312a52(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:12,724 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:12,724 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/C, priority=13, startTime=1733384472244; duration=0sec 2024-12-05T07:41:12,724 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:12,724 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:C 2024-12-05T07:41:12,725 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/938b7cbf4cb94643baabf15744adb2b9 2024-12-05T07:41:12,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/39d6951db1b141f193bab39b62635c63 is 50, key is test_row_0/B:col10/1733384471435/Put/seqid=0 2024-12-05T07:41:12,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384532736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:12,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742116_1292 (size=12301) 2024-12-05T07:41:12,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33160 deadline: 1733384532806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:12,808 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:12,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33158 deadline: 1733384532825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:12,827 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:12,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:12,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33180 deadline: 1733384532856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:12,859 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18314 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:12,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33200 deadline: 1733384532866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:12,868 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 063dfe74bcf23acd38effeda60a67fa8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18324 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:12,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:12,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384532946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:13,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T07:41:13,155 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/39d6951db1b141f193bab39b62635c63 2024-12-05T07:41:13,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/6a02717ac06141cc810c05d3f7b702af is 50, key is test_row_0/C:col10/1733384471435/Put/seqid=0 2024-12-05T07:41:13,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742117_1293 (size=12301) 2024-12-05T07:41:13,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384533256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:13,568 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/6a02717ac06141cc810c05d3f7b702af 2024-12-05T07:41:13,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/938b7cbf4cb94643baabf15744adb2b9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/938b7cbf4cb94643baabf15744adb2b9 2024-12-05T07:41:13,579 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/938b7cbf4cb94643baabf15744adb2b9, entries=150, sequenceid=352, filesize=12.0 K 2024-12-05T07:41:13,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/39d6951db1b141f193bab39b62635c63 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39d6951db1b141f193bab39b62635c63 2024-12-05T07:41:13,584 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39d6951db1b141f193bab39b62635c63, entries=150, sequenceid=352, filesize=12.0 K 2024-12-05T07:41:13,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/6a02717ac06141cc810c05d3f7b702af as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6a02717ac06141cc810c05d3f7b702af 2024-12-05T07:41:13,589 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6a02717ac06141cc810c05d3f7b702af, entries=150, sequenceid=352, filesize=12.0 K 2024-12-05T07:41:13,590 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 063dfe74bcf23acd38effeda60a67fa8 in 1274ms, sequenceid=352, compaction requested=false 2024-12-05T07:41:13,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:13,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:13,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-05T07:41:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-05T07:41:13,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-05T07:41:13,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5820 sec 2024-12-05T07:41:13,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.5880 sec 2024-12-05T07:41:13,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:13,768 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:13,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T07:41:13,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/a5c044bd89454efeaef1fad0730af3f1 is 50, key is test_row_0/A:col10/1733384472615/Put/seqid=0 2024-12-05T07:41:13,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742118_1294 (size=14741) 2024-12-05T07:41:13,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:13,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384533844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:13,931 INFO [Thread-1046 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1046. 2024-12-05T07:41:13,931 DEBUG [Thread-1046 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:13,931 DEBUG [Thread-1046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:13,931 INFO [Thread-1048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1048. 2024-12-05T07:41:13,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:13,931 DEBUG [Thread-1048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:13,931 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:13,931 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:13,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:13,932 DEBUG [Thread-1046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:13,933 INFO [Thread-1040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1040. 2024-12-05T07:41:13,933 DEBUG [Thread-1040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:13,933 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:13,933 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:13,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:13,934 INFO [Thread-1042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1042. 2024-12-05T07:41:13,934 DEBUG [Thread-1042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:13,934 DEBUG [Thread-1042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:13,934 DEBUG [Thread-1042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:13,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:13,936 INFO [Thread-1044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1044. 2024-12-05T07:41:13,936 DEBUG [Thread-1044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:13,936 DEBUG [Thread-1044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:13,936 DEBUG [Thread-1044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:13,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:13,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384533956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T07:41:14,157 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:14,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384534166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:14,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:14,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:14,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:14,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/a5c044bd89454efeaef1fad0730af3f1 2024-12-05T07:41:14,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/ea9bfb29963643e8bfbbfbdc1ae152b2 is 50, key is test_row_0/B:col10/1733384472615/Put/seqid=0 2024-12-05T07:41:14,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742119_1295 (size=12301) 2024-12-05T07:41:14,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384534476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:14,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:14,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/ea9bfb29963643e8bfbbfbdc1ae152b2 2024-12-05T07:41:14,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/f5ff447da9184594a1b8b7cbb159af78 is 50, key is test_row_0/C:col10/1733384472615/Put/seqid=0 2024-12-05T07:41:14,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742120_1296 (size=12301) 2024-12-05T07:41:14,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 337 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33248 deadline: 1733384534986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:15,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/f5ff447da9184594a1b8b7cbb159af78 2024-12-05T07:41:15,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/a5c044bd89454efeaef1fad0730af3f1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/a5c044bd89454efeaef1fad0730af3f1 2024-12-05T07:41:15,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/a5c044bd89454efeaef1fad0730af3f1, entries=200, sequenceid=368, filesize=14.4 K 2024-12-05T07:41:15,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/ea9bfb29963643e8bfbbfbdc1ae152b2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ea9bfb29963643e8bfbbfbdc1ae152b2 2024-12-05T07:41:15,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ea9bfb29963643e8bfbbfbdc1ae152b2, entries=150, sequenceid=368, filesize=12.0 K 2024-12-05T07:41:15,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/f5ff447da9184594a1b8b7cbb159af78 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/f5ff447da9184594a1b8b7cbb159af78 2024-12-05T07:41:15,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/f5ff447da9184594a1b8b7cbb159af78, entries=150, sequenceid=368, filesize=12.0 K 2024-12-05T07:41:15,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 063dfe74bcf23acd38effeda60a67fa8 in 1264ms, sequenceid=368, compaction requested=true 2024-12-05T07:41:15,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:15,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:15,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:15,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:15,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:15,033 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:15,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 063dfe74bcf23acd38effeda60a67fa8:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:15,033 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:15,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:15,034 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:15,034 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:15,034 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/B is initiating minor compaction (all files) 2024-12-05T07:41:15,034 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/A is initiating minor compaction (all files) 2024-12-05T07:41:15,034 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/B in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:15,034 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/A in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:15,034 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/3217b506f2014a00b843e75162b560b7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39d6951db1b141f193bab39b62635c63, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ea9bfb29963643e8bfbbfbdc1ae152b2] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=36.8 K 2024-12-05T07:41:15,034 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/84dc8192124a45838d24a699c0051e55, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/938b7cbf4cb94643baabf15744adb2b9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/a5c044bd89454efeaef1fad0730af3f1] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=39.2 K 2024-12-05T07:41:15,034 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3217b506f2014a00b843e75162b560b7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733384470703 2024-12-05T07:41:15,034 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 84dc8192124a45838d24a699c0051e55, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733384470703 2024-12-05T07:41:15,034 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39d6951db1b141f193bab39b62635c63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733384471395 2024-12-05T07:41:15,035 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 938b7cbf4cb94643baabf15744adb2b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733384471395 2024-12-05T07:41:15,035 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea9bfb29963643e8bfbbfbdc1ae152b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733384472599 2024-12-05T07:41:15,035 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a5c044bd89454efeaef1fad0730af3f1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733384472599 2024-12-05T07:41:15,040 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#B#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:15,041 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/bfdc6459b7e14d6794950f6f7191cc9f is 50, key is test_row_0/B:col10/1733384472615/Put/seqid=0 2024-12-05T07:41:15,041 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#A#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:15,041 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/89e63bf89c2c42aba8ea2cafc2620d57 is 50, key is test_row_0/A:col10/1733384472615/Put/seqid=0 2024-12-05T07:41:15,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742121_1297 (size=13187) 2024-12-05T07:41:15,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742122_1298 (size=13187) 2024-12-05T07:41:15,455 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/bfdc6459b7e14d6794950f6f7191cc9f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/bfdc6459b7e14d6794950f6f7191cc9f 2024-12-05T07:41:15,456 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/89e63bf89c2c42aba8ea2cafc2620d57 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/89e63bf89c2c42aba8ea2cafc2620d57 2024-12-05T07:41:15,460 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/A of 063dfe74bcf23acd38effeda60a67fa8 into 89e63bf89c2c42aba8ea2cafc2620d57(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:15,460 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/B of 063dfe74bcf23acd38effeda60a67fa8 into bfdc6459b7e14d6794950f6f7191cc9f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:15,460 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:15,460 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:15,460 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/B, priority=13, startTime=1733384475033; duration=0sec 2024-12-05T07:41:15,460 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/A, priority=13, startTime=1733384475033; duration=0sec 2024-12-05T07:41:15,460 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:15,460 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:A 2024-12-05T07:41:15,460 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:15,460 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:B 2024-12-05T07:41:15,460 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:15,461 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:15,461 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 063dfe74bcf23acd38effeda60a67fa8/C is initiating minor compaction (all files) 2024-12-05T07:41:15,461 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 063dfe74bcf23acd38effeda60a67fa8/C in TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:15,461 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/dd4cf6bd7e1f47b99cf0640fe0312a52, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6a02717ac06141cc810c05d3f7b702af, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/f5ff447da9184594a1b8b7cbb159af78] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp, totalSize=36.8 K 2024-12-05T07:41:15,461 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting dd4cf6bd7e1f47b99cf0640fe0312a52, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733384470703 2024-12-05T07:41:15,461 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a02717ac06141cc810c05d3f7b702af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1733384471395 2024-12-05T07:41:15,461 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f5ff447da9184594a1b8b7cbb159af78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733384472599 2024-12-05T07:41:15,466 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 063dfe74bcf23acd38effeda60a67fa8#C#compaction#256 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:15,467 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/309d05cb158e466eb71c3a998e3b5373 is 50, key is test_row_0/C:col10/1733384472615/Put/seqid=0 2024-12-05T07:41:15,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742123_1299 (size=13187) 2024-12-05T07:41:15,874 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/309d05cb158e466eb71c3a998e3b5373 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/309d05cb158e466eb71c3a998e3b5373 2024-12-05T07:41:15,879 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 063dfe74bcf23acd38effeda60a67fa8/C of 063dfe74bcf23acd38effeda60a67fa8 into 309d05cb158e466eb71c3a998e3b5373(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:15,879 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:15,879 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., storeName=063dfe74bcf23acd38effeda60a67fa8/C, priority=13, startTime=1733384475033; duration=0sec 2024-12-05T07:41:15,879 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:15,879 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 063dfe74bcf23acd38effeda60a67fa8:C 2024-12-05T07:41:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:16,009 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:16,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T07:41:16,009 INFO [Thread-1033 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1033. 2024-12-05T07:41:16,009 DEBUG [Thread-1033 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:16,009 DEBUG [Thread-1033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:16,010 DEBUG [Thread-1033 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:16,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:16,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/9f8d86290d534718ae8b72d2fbc4bc15 is 50, key is test_row_0/A:col10/1733384473831/Put/seqid=0 2024-12-05T07:41:16,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742124_1300 (size=12301) 2024-12-05T07:41:16,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/9f8d86290d534718ae8b72d2fbc4bc15 2024-12-05T07:41:16,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/3044138733e14e1ebe447538bb7fb853 is 50, key is test_row_0/B:col10/1733384473831/Put/seqid=0 2024-12-05T07:41:16,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742125_1301 (size=12301) 2024-12-05T07:41:16,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/3044138733e14e1ebe447538bb7fb853 2024-12-05T07:41:16,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/976455f8fe3f46a59afab371de0addc2 is 50, key is test_row_0/C:col10/1733384473831/Put/seqid=0 2024-12-05T07:41:16,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742126_1302 (size=12301) 2024-12-05T07:41:17,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/976455f8fe3f46a59afab371de0addc2 2024-12-05T07:41:17,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/9f8d86290d534718ae8b72d2fbc4bc15 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9f8d86290d534718ae8b72d2fbc4bc15 2024-12-05T07:41:17,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9f8d86290d534718ae8b72d2fbc4bc15, entries=150, sequenceid=394, filesize=12.0 K 2024-12-05T07:41:17,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/3044138733e14e1ebe447538bb7fb853 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/3044138733e14e1ebe447538bb7fb853 2024-12-05T07:41:17,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/3044138733e14e1ebe447538bb7fb853, entries=150, sequenceid=394, filesize=12.0 K 2024-12-05T07:41:17,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/976455f8fe3f46a59afab371de0addc2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/976455f8fe3f46a59afab371de0addc2 2024-12-05T07:41:17,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/976455f8fe3f46a59afab371de0addc2, entries=150, sequenceid=394, filesize=12.0 K 2024-12-05T07:41:17,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 063dfe74bcf23acd38effeda60a67fa8 in 1276ms, sequenceid=394, compaction requested=false 2024-12-05T07:41:17,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 063dfe74bcf23acd38effeda60a67fa8: 2024-12-05T07:41:22,574 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T07:41:22,841 INFO [Thread-1029 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1029. 2024-12-05T07:41:22,841 DEBUG [Thread-1029 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:22,841 DEBUG [Thread-1029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:22,842 DEBUG [Thread-1029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:22,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:22,909 INFO [Thread-1031 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1031. 2024-12-05T07:41:22,909 DEBUG [Thread-1031 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:22,909 DEBUG [Thread-1031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:22,909 DEBUG [Thread-1031 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:22,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:22,940 INFO [Thread-1037 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1037. 2024-12-05T07:41:22,940 DEBUG [Thread-1037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:22,940 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:22,940 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:22,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:22,971 INFO [Thread-1035 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1035. 2024-12-05T07:41:22,971 DEBUG [Thread-1035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:22,972 DEBUG [Thread-1035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:22,972 DEBUG [Thread-1035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:22,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 5 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 270 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5556 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5319 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5001 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5561 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5315 2024-12-05T07:41:22,973 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T07:41:22,974 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T07:41:22,974 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T07:41:22,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:22,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:22,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:22,976 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T07:41:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T07:41:22,980 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384482979"}]},"ts":"1733384482979"} 2024-12-05T07:41:22,981 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T07:41:22,981 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T07:41:22,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T07:41:22,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=063dfe74bcf23acd38effeda60a67fa8, UNASSIGN}] 2024-12-05T07:41:22,984 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=063dfe74bcf23acd38effeda60a67fa8, UNASSIGN 2024-12-05T07:41:22,985 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=063dfe74bcf23acd38effeda60a67fa8, regionState=CLOSING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:22,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=063dfe74bcf23acd38effeda60a67fa8, UNASSIGN because future has completed 2024-12-05T07:41:22,986 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T07:41:22,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:41:23,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T07:41:23,140 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:23,140 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T07:41:23,140 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing 063dfe74bcf23acd38effeda60a67fa8, disabling compactions & flushes 2024-12-05T07:41:23,140 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:23,141 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:23,141 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. after waiting 0 ms 2024-12-05T07:41:23,141 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:23,141 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing 063dfe74bcf23acd38effeda60a67fa8 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-05T07:41:23,149 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/ab67656c6a944fa29619119f183f91dc is 50, key is test_row_0/A:col10/1733384482907/Put/seqid=0 2024-12-05T07:41:23,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742127_1303 (size=12301) 2024-12-05T07:41:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T07:41:23,558 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/ab67656c6a944fa29619119f183f91dc 2024-12-05T07:41:23,571 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/545e0b0c0c7e43a2b779c26ee1409aa1 is 50, key is test_row_0/B:col10/1733384482907/Put/seqid=0 2024-12-05T07:41:23,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742128_1304 (size=12301) 2024-12-05T07:41:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T07:41:23,975 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/545e0b0c0c7e43a2b779c26ee1409aa1 2024-12-05T07:41:23,987 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/e3168380802a4bafb9fa8d152bd15070 is 50, key is test_row_0/C:col10/1733384482907/Put/seqid=0 2024-12-05T07:41:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742129_1305 (size=12301) 2024-12-05T07:41:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T07:41:24,392 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/e3168380802a4bafb9fa8d152bd15070 2024-12-05T07:41:24,443 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/A/ab67656c6a944fa29619119f183f91dc as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/ab67656c6a944fa29619119f183f91dc 2024-12-05T07:41:24,449 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/ab67656c6a944fa29619119f183f91dc, entries=150, sequenceid=401, filesize=12.0 K 2024-12-05T07:41:24,450 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/B/545e0b0c0c7e43a2b779c26ee1409aa1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/545e0b0c0c7e43a2b779c26ee1409aa1 2024-12-05T07:41:24,455 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/545e0b0c0c7e43a2b779c26ee1409aa1, entries=150, sequenceid=401, filesize=12.0 K 2024-12-05T07:41:24,456 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/.tmp/C/e3168380802a4bafb9fa8d152bd15070 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/e3168380802a4bafb9fa8d152bd15070 2024-12-05T07:41:24,459 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/e3168380802a4bafb9fa8d152bd15070, entries=150, sequenceid=401, filesize=12.0 K 2024-12-05T07:41:24,460 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 063dfe74bcf23acd38effeda60a67fa8 in 1319ms, sequenceid=401, compaction requested=true 2024-12-05T07:41:24,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/6b591e9affa748f39e8f9cba4df6f343, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/feda5ef151f44f0c85e140ac6f354b5e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/695519345e764cb7917d5072d2928227, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/4cbbad0771cb4a4e8a918cd1e60628df, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/44177e9f20ad4563a2df4604913507eb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e65523871d8245f3a7cd7a70d47ed6dc, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/5fcf6b0cc00e49e999f4b0fe565cbb87, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e038fe8b67d2480c80aca2e63fc95fcf, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/034c7d0eab484593b06282f581454f07, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9b3f4cd1d30a451e9ef707d82771242e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9f2c31a86108480092cb6affadf25b94, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/509ef611ad5f41ff921f40d68ab2e815, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/94be4a7379074a6da87a73a29c6f2506, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/77738a2a9d81410a9584186106c4736d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1630b053863d4d5cbaa80d192e92cdc5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/2c0b8c747a1d472e8a2f1cc93eeb3d71, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1510782636684e7dba420522182381ed, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/05d2e4c11e22482ca680eee4d126b538, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/505b07da413d4e658fe194a20604e60b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/326483f4eb594740bdee0de7f4665828, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/8a01f292661641d48abb5c43c5aaf79b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/aedc7f084de4427a9e08dc442d40ddfb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/84dc8192124a45838d24a699c0051e55, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/122a88ab73da4c38bb3a9cbec38e3d5b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/938b7cbf4cb94643baabf15744adb2b9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/a5c044bd89454efeaef1fad0730af3f1] to archive 2024-12-05T07:41:24,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:41:24,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/6b591e9affa748f39e8f9cba4df6f343 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/6b591e9affa748f39e8f9cba4df6f343 2024-12-05T07:41:24,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/feda5ef151f44f0c85e140ac6f354b5e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/feda5ef151f44f0c85e140ac6f354b5e 2024-12-05T07:41:24,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/695519345e764cb7917d5072d2928227 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/695519345e764cb7917d5072d2928227 2024-12-05T07:41:24,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/4cbbad0771cb4a4e8a918cd1e60628df to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/4cbbad0771cb4a4e8a918cd1e60628df 2024-12-05T07:41:24,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/44177e9f20ad4563a2df4604913507eb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/44177e9f20ad4563a2df4604913507eb 2024-12-05T07:41:24,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e65523871d8245f3a7cd7a70d47ed6dc to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e65523871d8245f3a7cd7a70d47ed6dc 2024-12-05T07:41:24,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/5fcf6b0cc00e49e999f4b0fe565cbb87 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/5fcf6b0cc00e49e999f4b0fe565cbb87 2024-12-05T07:41:24,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e038fe8b67d2480c80aca2e63fc95fcf to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/e038fe8b67d2480c80aca2e63fc95fcf 2024-12-05T07:41:24,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/034c7d0eab484593b06282f581454f07 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/034c7d0eab484593b06282f581454f07 2024-12-05T07:41:24,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9b3f4cd1d30a451e9ef707d82771242e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9b3f4cd1d30a451e9ef707d82771242e 2024-12-05T07:41:24,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9f2c31a86108480092cb6affadf25b94 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9f2c31a86108480092cb6affadf25b94 2024-12-05T07:41:24,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/509ef611ad5f41ff921f40d68ab2e815 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/509ef611ad5f41ff921f40d68ab2e815 2024-12-05T07:41:24,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/94be4a7379074a6da87a73a29c6f2506 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/94be4a7379074a6da87a73a29c6f2506 2024-12-05T07:41:24,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/77738a2a9d81410a9584186106c4736d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/77738a2a9d81410a9584186106c4736d 2024-12-05T07:41:24,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1630b053863d4d5cbaa80d192e92cdc5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1630b053863d4d5cbaa80d192e92cdc5 2024-12-05T07:41:24,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/2c0b8c747a1d472e8a2f1cc93eeb3d71 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/2c0b8c747a1d472e8a2f1cc93eeb3d71 2024-12-05T07:41:24,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1510782636684e7dba420522182381ed to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/1510782636684e7dba420522182381ed 2024-12-05T07:41:24,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/05d2e4c11e22482ca680eee4d126b538 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/05d2e4c11e22482ca680eee4d126b538 2024-12-05T07:41:24,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/505b07da413d4e658fe194a20604e60b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/505b07da413d4e658fe194a20604e60b 2024-12-05T07:41:24,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/326483f4eb594740bdee0de7f4665828 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/326483f4eb594740bdee0de7f4665828 2024-12-05T07:41:24,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/8a01f292661641d48abb5c43c5aaf79b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/8a01f292661641d48abb5c43c5aaf79b 2024-12-05T07:41:24,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/aedc7f084de4427a9e08dc442d40ddfb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/aedc7f084de4427a9e08dc442d40ddfb 2024-12-05T07:41:24,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/84dc8192124a45838d24a699c0051e55 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/84dc8192124a45838d24a699c0051e55 2024-12-05T07:41:24,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/122a88ab73da4c38bb3a9cbec38e3d5b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/122a88ab73da4c38bb3a9cbec38e3d5b 2024-12-05T07:41:24,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/938b7cbf4cb94643baabf15744adb2b9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/938b7cbf4cb94643baabf15744adb2b9 2024-12-05T07:41:24,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/a5c044bd89454efeaef1fad0730af3f1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/a5c044bd89454efeaef1fad0730af3f1 2024-12-05T07:41:24,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/d6bc644c8e4c4f6f964302f6f8f9f0c4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/29c30f6ae610498b8d536e5814237034, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/bffcd5c3434a48fc8e7f27541a4e7d8a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/a0a184ba5b71452b92f9e3b908b5fffc, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/904997487d3a42e38dca49c6b5dbafb5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/89b7bb68b8184d84b618e6b6f5918132, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/a80addb5af6c403185200192b96218b5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/6326d5eeff9b444ba88390f3705b5da5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/19751b6860914bb6adb743340b6d551d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/e690f87850d743f8846ed2b0d0cf6c0f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f2f30ff0f0604bff9e03daacf9f967a3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f042ef8f9db641eb9f9091f6cbcd5b0b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/883ff3f0d96346bc89bd85ef916c26d7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f1ab31be2ab443288d518773ace387f9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/733877547e7b4b2690a14bdce02e8306, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/e38e0e54ffd94237b4b339786acf02f2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/d287b92a138948fbb9d6a98d96e6b2da, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/7769c7efe81743bbbe2ea1d91d5e6ba6, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39e9563fbc7a49f98bc475469fb07180, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/00373c50b67d4c1aa301bf65f819c2eb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/50d921f2719a4602a3274a90d1dfac10, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/1cbc0ab232334e72b947051b967accd9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/3217b506f2014a00b843e75162b560b7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ba1ee0c0ea24466c90d78c231d8d8ed6, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39d6951db1b141f193bab39b62635c63, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ea9bfb29963643e8bfbbfbdc1ae152b2] to archive 2024-12-05T07:41:24,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:41:24,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/d6bc644c8e4c4f6f964302f6f8f9f0c4 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/d6bc644c8e4c4f6f964302f6f8f9f0c4 2024-12-05T07:41:24,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/29c30f6ae610498b8d536e5814237034 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/29c30f6ae610498b8d536e5814237034 2024-12-05T07:41:24,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/bffcd5c3434a48fc8e7f27541a4e7d8a to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/bffcd5c3434a48fc8e7f27541a4e7d8a 2024-12-05T07:41:24,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/a0a184ba5b71452b92f9e3b908b5fffc to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/a0a184ba5b71452b92f9e3b908b5fffc 2024-12-05T07:41:24,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/904997487d3a42e38dca49c6b5dbafb5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/904997487d3a42e38dca49c6b5dbafb5 2024-12-05T07:41:24,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/89b7bb68b8184d84b618e6b6f5918132 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/89b7bb68b8184d84b618e6b6f5918132 2024-12-05T07:41:24,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/a80addb5af6c403185200192b96218b5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/a80addb5af6c403185200192b96218b5 2024-12-05T07:41:24,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/6326d5eeff9b444ba88390f3705b5da5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/6326d5eeff9b444ba88390f3705b5da5 2024-12-05T07:41:24,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/19751b6860914bb6adb743340b6d551d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/19751b6860914bb6adb743340b6d551d 2024-12-05T07:41:24,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/e690f87850d743f8846ed2b0d0cf6c0f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/e690f87850d743f8846ed2b0d0cf6c0f 2024-12-05T07:41:24,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f2f30ff0f0604bff9e03daacf9f967a3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f2f30ff0f0604bff9e03daacf9f967a3 2024-12-05T07:41:24,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f042ef8f9db641eb9f9091f6cbcd5b0b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f042ef8f9db641eb9f9091f6cbcd5b0b 2024-12-05T07:41:24,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/883ff3f0d96346bc89bd85ef916c26d7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/883ff3f0d96346bc89bd85ef916c26d7 2024-12-05T07:41:24,507 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f1ab31be2ab443288d518773ace387f9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/f1ab31be2ab443288d518773ace387f9 2024-12-05T07:41:24,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/733877547e7b4b2690a14bdce02e8306 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/733877547e7b4b2690a14bdce02e8306 2024-12-05T07:41:24,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/e38e0e54ffd94237b4b339786acf02f2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/e38e0e54ffd94237b4b339786acf02f2 2024-12-05T07:41:24,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/d287b92a138948fbb9d6a98d96e6b2da to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/d287b92a138948fbb9d6a98d96e6b2da 2024-12-05T07:41:24,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/7769c7efe81743bbbe2ea1d91d5e6ba6 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/7769c7efe81743bbbe2ea1d91d5e6ba6 2024-12-05T07:41:24,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39e9563fbc7a49f98bc475469fb07180 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39e9563fbc7a49f98bc475469fb07180 2024-12-05T07:41:24,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/00373c50b67d4c1aa301bf65f819c2eb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/00373c50b67d4c1aa301bf65f819c2eb 2024-12-05T07:41:24,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/50d921f2719a4602a3274a90d1dfac10 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/50d921f2719a4602a3274a90d1dfac10 2024-12-05T07:41:24,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/1cbc0ab232334e72b947051b967accd9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/1cbc0ab232334e72b947051b967accd9 2024-12-05T07:41:24,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/3217b506f2014a00b843e75162b560b7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/3217b506f2014a00b843e75162b560b7 2024-12-05T07:41:24,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ba1ee0c0ea24466c90d78c231d8d8ed6 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ba1ee0c0ea24466c90d78c231d8d8ed6 2024-12-05T07:41:24,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39d6951db1b141f193bab39b62635c63 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/39d6951db1b141f193bab39b62635c63 2024-12-05T07:41:24,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ea9bfb29963643e8bfbbfbdc1ae152b2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/ea9bfb29963643e8bfbbfbdc1ae152b2 2024-12-05T07:41:24,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/80cf0af12d7e4f70ac16b0c2cd5d8e35, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/27c40958add64b4e9aae431d8bad6c8a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/dc06700c59e54ffba9b8760d88b62e3e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/5ba9eca2e093425bab752dc507b2b19c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/2f372829774f4da69bb3b2bf5e606772, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/50c9321023a94aed85835e351d192f92, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/d35d07afd74244e68b488ffeddf36f14, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/d7c9382cc8fe4c1ca9a5f8b9c22e3034, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7323abd34b7241d3b8d801b3cb7106dd, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/43332a71c24646e0a52aeb6388fad215, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7addcb64eb0d4d14beb84ee975fe4997, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/ba6065ae3ded4f1ca6cb3e7af85db098, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/59a200249b1f4f7684afbf12e0f0da5f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/95613bf1f01c4bae9a09a0473e1e6525, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/34819125e37c4601add11b757c9dc63c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4ce2a43ceb7a4f0b86667eda217117f3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/658f06dc2e11406e92d832d1f214cdd7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/9ed509feba95467eb93802607dfa8d72, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/b0d6bc0dac304bf284bc631aa4d07a58, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/c9a4f18c3cfe4a3ea7dc26c1134e16be, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4d645688561f43bc9e6f29d1fa525aaf, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/fcf5409df77b46acb68aed17d13eeaba, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/dd4cf6bd7e1f47b99cf0640fe0312a52, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6b2c7eee20c049ff8494791198c2764e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6a02717ac06141cc810c05d3f7b702af, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/f5ff447da9184594a1b8b7cbb159af78] to archive 2024-12-05T07:41:24,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:41:24,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/80cf0af12d7e4f70ac16b0c2cd5d8e35 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/80cf0af12d7e4f70ac16b0c2cd5d8e35 2024-12-05T07:41:24,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/27c40958add64b4e9aae431d8bad6c8a to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/27c40958add64b4e9aae431d8bad6c8a 2024-12-05T07:41:24,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/dc06700c59e54ffba9b8760d88b62e3e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/dc06700c59e54ffba9b8760d88b62e3e 2024-12-05T07:41:24,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/5ba9eca2e093425bab752dc507b2b19c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/5ba9eca2e093425bab752dc507b2b19c 2024-12-05T07:41:24,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/2f372829774f4da69bb3b2bf5e606772 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/2f372829774f4da69bb3b2bf5e606772 2024-12-05T07:41:24,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/50c9321023a94aed85835e351d192f92 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/50c9321023a94aed85835e351d192f92 2024-12-05T07:41:24,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/d35d07afd74244e68b488ffeddf36f14 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/d35d07afd74244e68b488ffeddf36f14 2024-12-05T07:41:24,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/d7c9382cc8fe4c1ca9a5f8b9c22e3034 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/d7c9382cc8fe4c1ca9a5f8b9c22e3034 2024-12-05T07:41:24,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7323abd34b7241d3b8d801b3cb7106dd to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7323abd34b7241d3b8d801b3cb7106dd 2024-12-05T07:41:24,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/43332a71c24646e0a52aeb6388fad215 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/43332a71c24646e0a52aeb6388fad215 2024-12-05T07:41:24,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7addcb64eb0d4d14beb84ee975fe4997 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/7addcb64eb0d4d14beb84ee975fe4997 2024-12-05T07:41:24,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/ba6065ae3ded4f1ca6cb3e7af85db098 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/ba6065ae3ded4f1ca6cb3e7af85db098 2024-12-05T07:41:24,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/59a200249b1f4f7684afbf12e0f0da5f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/59a200249b1f4f7684afbf12e0f0da5f 2024-12-05T07:41:24,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/95613bf1f01c4bae9a09a0473e1e6525 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/95613bf1f01c4bae9a09a0473e1e6525 2024-12-05T07:41:24,529 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/34819125e37c4601add11b757c9dc63c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/34819125e37c4601add11b757c9dc63c 2024-12-05T07:41:24,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4ce2a43ceb7a4f0b86667eda217117f3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4ce2a43ceb7a4f0b86667eda217117f3 2024-12-05T07:41:24,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/658f06dc2e11406e92d832d1f214cdd7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/658f06dc2e11406e92d832d1f214cdd7 2024-12-05T07:41:24,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/9ed509feba95467eb93802607dfa8d72 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/9ed509feba95467eb93802607dfa8d72 2024-12-05T07:41:24,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/b0d6bc0dac304bf284bc631aa4d07a58 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/b0d6bc0dac304bf284bc631aa4d07a58 2024-12-05T07:41:24,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/c9a4f18c3cfe4a3ea7dc26c1134e16be to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/c9a4f18c3cfe4a3ea7dc26c1134e16be 2024-12-05T07:41:24,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4d645688561f43bc9e6f29d1fa525aaf to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/4d645688561f43bc9e6f29d1fa525aaf 2024-12-05T07:41:24,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/fcf5409df77b46acb68aed17d13eeaba to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/fcf5409df77b46acb68aed17d13eeaba 2024-12-05T07:41:24,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/dd4cf6bd7e1f47b99cf0640fe0312a52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/dd4cf6bd7e1f47b99cf0640fe0312a52 2024-12-05T07:41:24,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6b2c7eee20c049ff8494791198c2764e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6b2c7eee20c049ff8494791198c2764e 2024-12-05T07:41:24,538 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6a02717ac06141cc810c05d3f7b702af to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/6a02717ac06141cc810c05d3f7b702af 2024-12-05T07:41:24,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/f5ff447da9184594a1b8b7cbb159af78 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/f5ff447da9184594a1b8b7cbb159af78 2024-12-05T07:41:24,543 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/recovered.edits/404.seqid, newMaxSeqId=404, maxSeqId=1 2024-12-05T07:41:24,543 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. 2024-12-05T07:41:24,543 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for 063dfe74bcf23acd38effeda60a67fa8: Waiting for close lock at 1733384483140Running coprocessor pre-close hooks at 1733384483140Disabling compacts and flushes for region at 1733384483140Disabling writes for close at 1733384483141 (+1 ms)Obtaining lock to block concurrent updates at 1733384483141Preparing flush snapshotting stores in 063dfe74bcf23acd38effeda60a67fa8 at 1733384483141Finished memstore snapshotting TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1733384483141Flushing stores of TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8. at 1733384483142 (+1 ms)Flushing 063dfe74bcf23acd38effeda60a67fa8/A: creating writer at 1733384483142Flushing 063dfe74bcf23acd38effeda60a67fa8/A: appending metadata at 1733384483148 (+6 ms)Flushing 063dfe74bcf23acd38effeda60a67fa8/A: closing flushed file at 1733384483149 (+1 ms)Flushing 063dfe74bcf23acd38effeda60a67fa8/B: creating writer at 1733384483568 (+419 ms)Flushing 063dfe74bcf23acd38effeda60a67fa8/B: appending metadata at 1733384483570 (+2 ms)Flushing 063dfe74bcf23acd38effeda60a67fa8/B: closing flushed file at 1733384483570Flushing 063dfe74bcf23acd38effeda60a67fa8/C: creating writer at 1733384483982 (+412 ms)Flushing 063dfe74bcf23acd38effeda60a67fa8/C: appending metadata at 1733384483987 (+5 ms)Flushing 063dfe74bcf23acd38effeda60a67fa8/C: closing flushed file at 1733384483987Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@403724d8: reopening flushed file at 1733384484442 (+455 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4eccd42c: reopening flushed file at 1733384484449 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@292023c7: reopening flushed file at 1733384484455 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 063dfe74bcf23acd38effeda60a67fa8 in 1319ms, sequenceid=401, compaction requested=true at 1733384484460 (+5 ms)Writing region close event to WAL at 1733384484540 (+80 ms)Running coprocessor post-close hooks at 1733384484543 (+3 ms)Closed at 1733384484543 2024-12-05T07:41:24,545 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed 063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:24,545 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=063dfe74bcf23acd38effeda60a67fa8, regionState=CLOSED 2024-12-05T07:41:24,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:41:24,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-05T07:41:24,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure 063dfe74bcf23acd38effeda60a67fa8, server=fc6cd38557f3,36693,1733384394401 in 1.5600 sec 2024-12-05T07:41:24,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-05T07:41:24,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=063dfe74bcf23acd38effeda60a67fa8, UNASSIGN in 1.5660 sec 2024-12-05T07:41:24,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-05T07:41:24,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5690 sec 2024-12-05T07:41:24,553 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384484553"}]},"ts":"1733384484553"} 2024-12-05T07:41:24,554 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T07:41:24,554 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T07:41:24,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5790 sec 2024-12-05T07:41:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T07:41:25,137 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:25,139 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T07:41:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:25,144 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:25,146 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:25,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T07:41:25,150 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:25,153 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/recovered.edits] 2024-12-05T07:41:25,155 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/89e63bf89c2c42aba8ea2cafc2620d57 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/89e63bf89c2c42aba8ea2cafc2620d57 2024-12-05T07:41:25,157 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9f8d86290d534718ae8b72d2fbc4bc15 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/9f8d86290d534718ae8b72d2fbc4bc15 2024-12-05T07:41:25,158 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/ab67656c6a944fa29619119f183f91dc to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/A/ab67656c6a944fa29619119f183f91dc 2024-12-05T07:41:25,160 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/3044138733e14e1ebe447538bb7fb853 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/3044138733e14e1ebe447538bb7fb853 2024-12-05T07:41:25,162 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/545e0b0c0c7e43a2b779c26ee1409aa1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/545e0b0c0c7e43a2b779c26ee1409aa1 2024-12-05T07:41:25,163 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/bfdc6459b7e14d6794950f6f7191cc9f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/B/bfdc6459b7e14d6794950f6f7191cc9f 2024-12-05T07:41:25,166 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/309d05cb158e466eb71c3a998e3b5373 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/309d05cb158e466eb71c3a998e3b5373 2024-12-05T07:41:25,168 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/976455f8fe3f46a59afab371de0addc2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/976455f8fe3f46a59afab371de0addc2 2024-12-05T07:41:25,169 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/e3168380802a4bafb9fa8d152bd15070 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/C/e3168380802a4bafb9fa8d152bd15070 2024-12-05T07:41:25,173 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/recovered.edits/404.seqid to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8/recovered.edits/404.seqid 2024-12-05T07:41:25,174 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/063dfe74bcf23acd38effeda60a67fa8 2024-12-05T07:41:25,174 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T07:41:25,178 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:25,181 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T07:41:25,183 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T07:41:25,185 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:25,185 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T07:41:25,185 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733384485185"}]},"ts":"9223372036854775807"} 2024-12-05T07:41:25,186 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T07:41:25,187 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 063dfe74bcf23acd38effeda60a67fa8, NAME => 'TestAcidGuarantees,,1733384451664.063dfe74bcf23acd38effeda60a67fa8.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T07:41:25,187 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T07:41:25,187 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733384485187"}]},"ts":"9223372036854775807"} 2024-12-05T07:41:25,189 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T07:41:25,189 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:25,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-12-05T07:41:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T07:41:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T07:41:25,257 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:25,271 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 245), OpenFileDescriptor=451 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=501 (was 505), ProcessCount=11 (was 11), AvailableMemoryMB=6790 (was 6359) - AvailableMemoryMB LEAK? - 2024-12-05T07:41:25,279 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=501, ProcessCount=11, AvailableMemoryMB=6790 2024-12-05T07:41:25,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T07:41:25,280 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T07:41:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:25,282 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T07:41:25,282 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:25,282 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-12-05T07:41:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T07:41:25,283 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T07:41:25,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742130_1306 (size=959) 2024-12-05T07:41:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T07:41:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T07:41:25,696 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15 2024-12-05T07:41:25,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742131_1307 (size=53) 2024-12-05T07:41:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T07:41:26,107 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:41:26,107 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2d9b639d709a2e4f040a3e7eadeae633, disabling compactions & flushes 2024-12-05T07:41:26,107 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:26,107 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:26,107 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. after waiting 0 ms 2024-12-05T07:41:26,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:26,108 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:26,108 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2d9b639d709a2e4f040a3e7eadeae633: Waiting for close lock at 1733384486107Disabling compacts and flushes for region at 1733384486107Disabling writes for close at 1733384486107Writing region close event to WAL at 1733384486108 (+1 ms)Closed at 1733384486108 2024-12-05T07:41:26,110 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T07:41:26,111 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733384486110"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733384486110"}]},"ts":"1733384486110"} 2024-12-05T07:41:26,113 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T07:41:26,115 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T07:41:26,115 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384486115"}]},"ts":"1733384486115"} 2024-12-05T07:41:26,117 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T07:41:26,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, ASSIGN}] 2024-12-05T07:41:26,118 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, ASSIGN 2024-12-05T07:41:26,119 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, ASSIGN; state=OFFLINE, location=fc6cd38557f3,36693,1733384394401; forceNewPlan=false, retain=false 2024-12-05T07:41:26,270 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=2d9b639d709a2e4f040a3e7eadeae633, regionState=OPENING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:26,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, ASSIGN because future has completed 2024-12-05T07:41:26,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T07:41:26,432 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:26,432 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} 2024-12-05T07:41:26,433 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,433 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:41:26,433 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,433 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,434 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,435 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d9b639d709a2e4f040a3e7eadeae633 columnFamilyName A 2024-12-05T07:41:26,435 DEBUG [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:26,436 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(327): Store=2d9b639d709a2e4f040a3e7eadeae633/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:41:26,436 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,436 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d9b639d709a2e4f040a3e7eadeae633 columnFamilyName B 2024-12-05T07:41:26,436 DEBUG [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:26,437 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(327): Store=2d9b639d709a2e4f040a3e7eadeae633/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:41:26,437 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,438 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d9b639d709a2e4f040a3e7eadeae633 columnFamilyName C 2024-12-05T07:41:26,438 DEBUG [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:26,438 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(327): Store=2d9b639d709a2e4f040a3e7eadeae633/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:41:26,438 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,439 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,439 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,440 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,440 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,441 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T07:41:26,442 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,444 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T07:41:26,445 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened 2d9b639d709a2e4f040a3e7eadeae633; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60250598, jitterRate=-0.10219612717628479}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T07:41:26,445 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:26,446 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for 2d9b639d709a2e4f040a3e7eadeae633: Running coprocessor pre-open hook at 1733384486433Writing region info on filesystem at 1733384486433Initializing all the Stores at 1733384486434 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384486434Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384486434Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384486434Cleaning up temporary data from old regions at 1733384486441 (+7 ms)Running coprocessor post-open hooks at 1733384486445 (+4 ms)Region opened successfully at 1733384486445 2024-12-05T07:41:26,446 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., pid=83, masterSystemTime=1733384486430 2024-12-05T07:41:26,448 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:26,448 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:26,449 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=2d9b639d709a2e4f040a3e7eadeae633, regionState=OPEN, openSeqNum=2, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:26,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:41:26,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-05T07:41:26,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 in 175 msec 2024-12-05T07:41:26,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-05T07:41:26,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, ASSIGN in 336 msec 2024-12-05T07:41:26,456 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T07:41:26,456 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384486456"}]},"ts":"1733384486456"} 2024-12-05T07:41:26,458 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T07:41:26,459 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T07:41:26,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-12-05T07:41:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T07:41:27,437 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:27,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f5e8d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:27,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:27,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:27,439 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15448242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:27,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:27,442 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3495d370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:27,443 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:27,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:27,447 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T07:41:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fc6cd38557f3,33643,1733384393481 2024-12-05T07:41:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61d13d33 2024-12-05T07:41:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T07:41:27,449 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T07:41:27,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T07:41:27,451 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T07:41:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:27,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742132_1308 (size=995) 2024-12-05T07:41:27,866 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-05T07:41:27,866 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-05T07:41:27,871 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T07:41:27,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, REOPEN/MOVE}] 2024-12-05T07:41:27,875 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, REOPEN/MOVE 2024-12-05T07:41:27,876 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=2d9b639d709a2e4f040a3e7eadeae633, regionState=CLOSING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:27,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, REOPEN/MOVE because future has completed 2024-12-05T07:41:27,878 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T07:41:27,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:41:28,031 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,031 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T07:41:28,031 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 2d9b639d709a2e4f040a3e7eadeae633, disabling compactions & flushes 2024-12-05T07:41:28,031 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,031 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,031 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. after waiting 0 ms 2024-12-05T07:41:28,031 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,040 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T07:41:28,041 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,041 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 2d9b639d709a2e4f040a3e7eadeae633: Waiting for close lock at 1733384488031Running coprocessor pre-close hooks at 1733384488031Disabling compacts and flushes for region at 1733384488031Disabling writes for close at 1733384488031Writing region close event to WAL at 1733384488032 (+1 ms)Running coprocessor post-close hooks at 1733384488040 (+8 ms)Closed at 1733384488041 (+1 ms) 2024-12-05T07:41:28,041 WARN [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: 2d9b639d709a2e4f040a3e7eadeae633 to self. 2024-12-05T07:41:28,043 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,043 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=2d9b639d709a2e4f040a3e7eadeae633, regionState=CLOSED 2024-12-05T07:41:28,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:41:28,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-05T07:41:28,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 in 167 msec 2024-12-05T07:41:28,047 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, REOPEN/MOVE; state=CLOSED, location=fc6cd38557f3,36693,1733384394401; forceNewPlan=false, retain=true 2024-12-05T07:41:28,199 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=2d9b639d709a2e4f040a3e7eadeae633, regionState=OPENING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, REOPEN/MOVE because future has completed 2024-12-05T07:41:28,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:41:28,361 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,362 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} 2024-12-05T07:41:28,362 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,362 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:41:28,363 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,363 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,365 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,367 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d9b639d709a2e4f040a3e7eadeae633 columnFamilyName A 2024-12-05T07:41:28,369 DEBUG [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:28,370 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(327): Store=2d9b639d709a2e4f040a3e7eadeae633/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:41:28,371 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,373 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d9b639d709a2e4f040a3e7eadeae633 columnFamilyName B 2024-12-05T07:41:28,373 DEBUG [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:28,374 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(327): Store=2d9b639d709a2e4f040a3e7eadeae633/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:41:28,374 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,375 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d9b639d709a2e4f040a3e7eadeae633 columnFamilyName C 2024-12-05T07:41:28,375 DEBUG [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:28,376 INFO [StoreOpener-2d9b639d709a2e4f040a3e7eadeae633-1 {}] regionserver.HStore(327): Store=2d9b639d709a2e4f040a3e7eadeae633/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:41:28,376 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,377 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,378 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,379 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,379 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,379 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T07:41:28,380 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,381 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 2d9b639d709a2e4f040a3e7eadeae633; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63778796, jitterRate=-0.04962188005447388}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T07:41:28,381 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,381 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 2d9b639d709a2e4f040a3e7eadeae633: Running coprocessor pre-open hook at 1733384488363Writing region info on filesystem at 1733384488363Initializing all the Stores at 1733384488364 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384488364Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384488365 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384488365Cleaning up temporary data from old regions at 1733384488379 (+14 ms)Running coprocessor post-open hooks at 1733384488381 (+2 ms)Region opened successfully at 1733384488381 2024-12-05T07:41:28,382 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., pid=88, masterSystemTime=1733384488356 2024-12-05T07:41:28,383 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,384 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,384 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=2d9b639d709a2e4f040a3e7eadeae633, regionState=OPEN, openSeqNum=5, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:41:28,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-12-05T07:41:28,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 in 182 msec 2024-12-05T07:41:28,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-05T07:41:28,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, REOPEN/MOVE in 513 msec 2024-12-05T07:41:28,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-05T07:41:28,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 518 msec 2024-12-05T07:41:28,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 938 msec 2024-12-05T07:41:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-05T07:41:28,392 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:28,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:28,392 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:28,393 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61191270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:28,395 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@304df757, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,396 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:28,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:28,397 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1945101, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:28,398 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a92b487, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:28,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:28,400 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:28,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:28,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49618aac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:28,402 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc41a13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:28,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:28,404 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a81a785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:28,405 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4398b7bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:28,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:28,408 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51e009cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:28,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:28,409 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:28,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9ea1d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:28,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:28,412 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:28,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e518629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:28,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:28,414 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:28,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2ab7cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:28,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:28,417 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a210db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:28,418 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:28,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@657e27f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:28,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:28,420 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40a54f3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:28,422 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:28,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f11f311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:28,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:28,424 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625675f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:28,426 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e21666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5135eed0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,427 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:28,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:28,428 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41456eca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:28,430 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ffed555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:28,431 DEBUG [Thread-1434 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:28,431 DEBUG [Thread-1436 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:28,431 DEBUG [Thread-1430 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:28,432 DEBUG [Thread-1438 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:28,432 DEBUG [Thread-1421 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:28,432 DEBUG [Thread-1419 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:28,432 DEBUG [Thread-1423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:28,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:28,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:28,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:28,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:28,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:28,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:28,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:28,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-05T07:41:28,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:28,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:41:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:41:28,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:28,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:41:28,438 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:28,439 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:41:28,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:41:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T07:41:28,440 DEBUG [Thread-1425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:41:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:41:28,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:28,440 DEBUG [Thread-1427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:28,440 DEBUG [Thread-1432 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:28,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:28,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:28,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:28,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:41:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:41:28,445 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:41:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,451 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:28,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:41:28,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056f8227b933634426b202a0564d97420e_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384488449/Put/seqid=0 2024-12-05T07:41:28,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742133_1309 (size=12154) 2024-12-05T07:41:28,485 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:28,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384548488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384548491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056f8227b933634426b202a0564d97420e_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056f8227b933634426b202a0564d97420e_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:28,498 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/69e9f21441c14e6896c0af27f6e61a70, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:28,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/69e9f21441c14e6896c0af27f6e61a70 is 175, key is test_row_0/A:col10/1733384488449/Put/seqid=0 2024-12-05T07:41:28,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384548495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384548496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384548496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742134_1310 (size=30955) 2024-12-05T07:41:28,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/69e9f21441c14e6896c0af27f6e61a70 2024-12-05T07:41:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T07:41:28,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/78bfe145415446b09e868f262f62bafd is 50, key is test_row_0/B:col10/1733384488449/Put/seqid=0 2024-12-05T07:41:28,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742135_1311 (size=12001) 2024-12-05T07:41:28,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/78bfe145415446b09e868f262f62bafd 2024-12-05T07:41:28,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/98b80eb7839343319f9956c2390aaae1 is 50, key is test_row_0/C:col10/1733384488449/Put/seqid=0 2024-12-05T07:41:28,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T07:41:28,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:28,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:28,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:28,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742136_1312 (size=12001) 2024-12-05T07:41:28,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384548605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384548606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384548606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384548616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384548616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T07:41:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:28,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T07:41:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384548816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384548816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384548816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384548826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384548826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:28,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:28,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T07:41:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:28,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:28,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:28,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:29,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/98b80eb7839343319f9956c2390aaae1 2024-12-05T07:41:29,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/69e9f21441c14e6896c0af27f6e61a70 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/69e9f21441c14e6896c0af27f6e61a70 2024-12-05T07:41:29,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/69e9f21441c14e6896c0af27f6e61a70, entries=150, sequenceid=15, filesize=30.2 K 2024-12-05T07:41:29,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/78bfe145415446b09e868f262f62bafd as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/78bfe145415446b09e868f262f62bafd 2024-12-05T07:41:29,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/78bfe145415446b09e868f262f62bafd, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T07:41:29,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/98b80eb7839343319f9956c2390aaae1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/98b80eb7839343319f9956c2390aaae1 2024-12-05T07:41:29,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/98b80eb7839343319f9956c2390aaae1, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T07:41:29,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 2d9b639d709a2e4f040a3e7eadeae633 in 565ms, sequenceid=15, compaction requested=false 2024-12-05T07:41:29,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:29,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T07:41:29,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:29,054 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T07:41:29,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205353f0476124e4464af419ae12a1ca524_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384488494/Put/seqid=0 2024-12-05T07:41:29,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742137_1313 (size=12154) 2024-12-05T07:41:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T07:41:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:29,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:29,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384549135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384549135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384549136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384549136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384549139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384549246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384549246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384549246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384549455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384549455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384549456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:29,518 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205353f0476124e4464af419ae12a1ca524_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205353f0476124e4464af419ae12a1ca524_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:29,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/43edd6263e534f98bd50b1042d14f794, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:29,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/43edd6263e534f98bd50b1042d14f794 is 175, key is test_row_0/A:col10/1733384488494/Put/seqid=0 2024-12-05T07:41:29,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742138_1314 (size=30955) 2024-12-05T07:41:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T07:41:29,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384549646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384549646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384549766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384549766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384549766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:29,922 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/43edd6263e534f98bd50b1042d14f794 2024-12-05T07:41:29,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/f5c7a207c86a479693964231d577ddf1 is 50, key is test_row_0/B:col10/1733384488494/Put/seqid=0 2024-12-05T07:41:29,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742139_1315 (size=12001) 2024-12-05T07:41:30,067 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T07:41:30,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384550276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:30,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384550276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:30,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384550276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:30,332 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/f5c7a207c86a479693964231d577ddf1 2024-12-05T07:41:30,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/ae7e57c29a7641298f309a9ae762560d is 50, key is test_row_0/C:col10/1733384488494/Put/seqid=0 2024-12-05T07:41:30,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742140_1316 (size=12001) 2024-12-05T07:41:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T07:41:30,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384550656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:30,660 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 2d9b639d709a2e4f040a3e7eadeae633 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:30,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:30,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:30,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384550666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:30,670 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 2d9b639d709a2e4f040a3e7eadeae633 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:30,744 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/ae7e57c29a7641298f309a9ae762560d 2024-12-05T07:41:30,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/43edd6263e534f98bd50b1042d14f794 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/43edd6263e534f98bd50b1042d14f794 2024-12-05T07:41:30,751 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/43edd6263e534f98bd50b1042d14f794, entries=150, sequenceid=41, filesize=30.2 K 2024-12-05T07:41:30,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/f5c7a207c86a479693964231d577ddf1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f5c7a207c86a479693964231d577ddf1 2024-12-05T07:41:30,756 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f5c7a207c86a479693964231d577ddf1, entries=150, sequenceid=41, filesize=11.7 K 2024-12-05T07:41:30,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/ae7e57c29a7641298f309a9ae762560d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/ae7e57c29a7641298f309a9ae762560d 2024-12-05T07:41:30,760 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/ae7e57c29a7641298f309a9ae762560d, entries=150, sequenceid=41, filesize=11.7 K 2024-12-05T07:41:30,761 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 2d9b639d709a2e4f040a3e7eadeae633 in 1707ms, sequenceid=41, compaction requested=false 2024-12-05T07:41:30,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:30,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:30,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-05T07:41:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-05T07:41:30,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-05T07:41:30,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3240 sec 2024-12-05T07:41:30,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.3300 sec 2024-12-05T07:41:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:31,290 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:31,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T07:41:31,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056a69602608e04824add89f964a4bd6c7_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384491289/Put/seqid=0 2024-12-05T07:41:31,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742141_1317 (size=12154) 2024-12-05T07:41:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384551336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:31,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384551336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384551337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:31,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384551446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384551446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:31,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:31,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384551455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:31,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384551656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384551656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:31,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384551667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:31,699 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:31,702 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056a69602608e04824add89f964a4bd6c7_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056a69602608e04824add89f964a4bd6c7_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:31,703 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/8f57610c73cb43f7aaae0d50453dae42, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:31,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/8f57610c73cb43f7aaae0d50453dae42 is 175, key is test_row_0/A:col10/1733384491289/Put/seqid=0 2024-12-05T07:41:31,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742142_1318 (size=30955) 2024-12-05T07:41:31,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384551966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384551966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:31,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384551976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:31,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:32,107 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/8f57610c73cb43f7aaae0d50453dae42 2024-12-05T07:41:32,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/fc874f2b32fa4a4fb294369ce2e93dbc is 50, key is test_row_0/B:col10/1733384491289/Put/seqid=0 2024-12-05T07:41:32,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742143_1319 (size=12001) 2024-12-05T07:41:32,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/fc874f2b32fa4a4fb294369ce2e93dbc 2024-12-05T07:41:32,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/c5837530abfd4b4eb9bc3a9cca5a5f44 is 50, key is test_row_0/C:col10/1733384491289/Put/seqid=0 2024-12-05T07:41:32,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742144_1320 (size=12001) 2024-12-05T07:41:32,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384552476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:32,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384552476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:32,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384552486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:32,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/c5837530abfd4b4eb9bc3a9cca5a5f44 2024-12-05T07:41:32,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/8f57610c73cb43f7aaae0d50453dae42 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8f57610c73cb43f7aaae0d50453dae42 2024-12-05T07:41:32,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8f57610c73cb43f7aaae0d50453dae42, entries=150, sequenceid=53, filesize=30.2 K 2024-12-05T07:41:32,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/fc874f2b32fa4a4fb294369ce2e93dbc as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/fc874f2b32fa4a4fb294369ce2e93dbc 2024-12-05T07:41:32,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/fc874f2b32fa4a4fb294369ce2e93dbc, entries=150, sequenceid=53, filesize=11.7 K 2024-12-05T07:41:32,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/c5837530abfd4b4eb9bc3a9cca5a5f44 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/c5837530abfd4b4eb9bc3a9cca5a5f44 2024-12-05T07:41:32,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/c5837530abfd4b4eb9bc3a9cca5a5f44, entries=150, sequenceid=53, filesize=11.7 K 2024-12-05T07:41:32,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2d9b639d709a2e4f040a3e7eadeae633 in 1260ms, sequenceid=53, compaction requested=true 2024-12-05T07:41:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:32,550 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:32,550 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:32,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:32,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:32,552 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:32,552 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/A is initiating minor compaction (all files) 2024-12-05T07:41:32,552 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:32,552 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/A in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:32,552 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/B is initiating minor compaction (all files) 2024-12-05T07:41:32,552 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/B in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:32,552 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/69e9f21441c14e6896c0af27f6e61a70, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/43edd6263e534f98bd50b1042d14f794, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8f57610c73cb43f7aaae0d50453dae42] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=90.7 K 2024-12-05T07:41:32,552 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/78bfe145415446b09e868f262f62bafd, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f5c7a207c86a479693964231d577ddf1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/fc874f2b32fa4a4fb294369ce2e93dbc] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=35.2 K 2024-12-05T07:41:32,552 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:32,552 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/69e9f21441c14e6896c0af27f6e61a70, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/43edd6263e534f98bd50b1042d14f794, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8f57610c73cb43f7aaae0d50453dae42] 2024-12-05T07:41:32,553 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69e9f21441c14e6896c0af27f6e61a70, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733384488445 2024-12-05T07:41:32,553 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 78bfe145415446b09e868f262f62bafd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733384488445 2024-12-05T07:41:32,553 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f5c7a207c86a479693964231d577ddf1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733384488485 2024-12-05T07:41:32,553 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43edd6263e534f98bd50b1042d14f794, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733384488485 2024-12-05T07:41:32,554 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting fc874f2b32fa4a4fb294369ce2e93dbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733384489130 2024-12-05T07:41:32,554 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f57610c73cb43f7aaae0d50453dae42, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733384489130 2024-12-05T07:41:32,559 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#B#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:32,559 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:32,560 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/f83590927ee349ba92fff1429ad0f5f2 is 50, key is test_row_0/B:col10/1733384491289/Put/seqid=0 2024-12-05T07:41:32,561 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412053fe8cc52e73b4e968beb6d5d42caf28e_2d9b639d709a2e4f040a3e7eadeae633 store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:32,563 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412053fe8cc52e73b4e968beb6d5d42caf28e_2d9b639d709a2e4f040a3e7eadeae633, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:32,563 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053fe8cc52e73b4e968beb6d5d42caf28e_2d9b639d709a2e4f040a3e7eadeae633 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:32,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742145_1321 (size=12104) 2024-12-05T07:41:32,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742146_1322 (size=4469) 2024-12-05T07:41:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T07:41:32,596 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:32,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-05T07:41:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T07:41:32,599 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:32,600 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:32,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:32,689 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:32,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T07:41:32,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120508a5e8959df1401a9bc9a2cef20a9733_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384492688/Put/seqid=0 2024-12-05T07:41:32,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742147_1323 (size=14594) 2024-12-05T07:41:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T07:41:32,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384552713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:32,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384552716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:32,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T07:41:32,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:32,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:32,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:32,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:32,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384552826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:32,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384552826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:32,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:32,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:32,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:32,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T07:41:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:32,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:32,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T07:41:32,968 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#A#compaction#273 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:32,969 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/c0c5d14d36344acdb997d18467df9b57 is 175, key is test_row_0/A:col10/1733384491289/Put/seqid=0 2024-12-05T07:41:32,971 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/f83590927ee349ba92fff1429ad0f5f2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f83590927ee349ba92fff1429ad0f5f2 2024-12-05T07:41:32,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742148_1324 (size=31058) 2024-12-05T07:41:32,978 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/c0c5d14d36344acdb997d18467df9b57 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/c0c5d14d36344acdb997d18467df9b57 2024-12-05T07:41:32,979 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/B of 2d9b639d709a2e4f040a3e7eadeae633 into f83590927ee349ba92fff1429ad0f5f2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:32,979 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:32,979 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/B, priority=13, startTime=1733384492550; duration=0sec 2024-12-05T07:41:32,979 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:32,979 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:B 2024-12-05T07:41:32,979 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:32,980 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:32,980 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/C is initiating minor compaction (all files) 2024-12-05T07:41:32,980 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/C in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:32,980 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/98b80eb7839343319f9956c2390aaae1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/ae7e57c29a7641298f309a9ae762560d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/c5837530abfd4b4eb9bc3a9cca5a5f44] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=35.2 K 2024-12-05T07:41:32,980 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 98b80eb7839343319f9956c2390aaae1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733384488445 2024-12-05T07:41:32,980 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting ae7e57c29a7641298f309a9ae762560d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1733384488485 2024-12-05T07:41:32,981 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting c5837530abfd4b4eb9bc3a9cca5a5f44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733384489130 2024-12-05T07:41:32,982 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/A of 2d9b639d709a2e4f040a3e7eadeae633 into c0c5d14d36344acdb997d18467df9b57(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:32,982 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:32,982 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/A, priority=13, startTime=1733384492550; duration=0sec 2024-12-05T07:41:32,982 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:32,982 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:A 2024-12-05T07:41:32,987 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#C#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:32,987 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/43eb62f7a0074a079b17730bff9ddc9d is 50, key is test_row_0/C:col10/1733384491289/Put/seqid=0 2024-12-05T07:41:32,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742149_1325 (size=12104) 2024-12-05T07:41:33,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384553036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:33,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384553036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:33,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T07:41:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,101 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:33,104 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120508a5e8959df1401a9bc9a2cef20a9733_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120508a5e8959df1401a9bc9a2cef20a9733_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:33,105 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/7b7d060e22f645dcb8e93d38b94d396b, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:33,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/7b7d060e22f645dcb8e93d38b94d396b is 175, key is test_row_0/A:col10/1733384492688/Put/seqid=0 2024-12-05T07:41:33,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742150_1326 (size=39549) 2024-12-05T07:41:33,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T07:41:33,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:33,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T07:41:33,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384553346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:33,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384553346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:33,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T07:41:33,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:33,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,394 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/43eb62f7a0074a079b17730bff9ddc9d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/43eb62f7a0074a079b17730bff9ddc9d 2024-12-05T07:41:33,398 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/C of 2d9b639d709a2e4f040a3e7eadeae633 into 43eb62f7a0074a079b17730bff9ddc9d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:33,398 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:33,398 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/C, priority=13, startTime=1733384492550; duration=0sec 2024-12-05T07:41:33,398 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:33,398 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:C 2024-12-05T07:41:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384553496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384553496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384553496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:33,499 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 2d9b639d709a2e4f040a3e7eadeae633 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:33,500 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 2d9b639d709a2e4f040a3e7eadeae633 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:33,499 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 2d9b639d709a2e4f040a3e7eadeae633 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:33,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/7b7d060e22f645dcb8e93d38b94d396b 2024-12-05T07:41:33,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T07:41:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:33,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/9fc91ecee8824c2695d228c11bd2fda1 is 50, key is test_row_0/B:col10/1733384492688/Put/seqid=0 2024-12-05T07:41:33,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742151_1327 (size=12001) 2024-12-05T07:41:33,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T07:41:33,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:33,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T07:41:33,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T07:41:33,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:33,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:33,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384553856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:33,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:33,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384553856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:33,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:33,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:33,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:33,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/9fc91ecee8824c2695d228c11bd2fda1 2024-12-05T07:41:33,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/87b74ce7afe54c688ff07f7a58d3de75 is 50, key is test_row_0/C:col10/1733384492688/Put/seqid=0 2024-12-05T07:41:33,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742152_1328 (size=12001) 2024-12-05T07:41:33,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/87b74ce7afe54c688ff07f7a58d3de75 2024-12-05T07:41:33,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/7b7d060e22f645dcb8e93d38b94d396b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/7b7d060e22f645dcb8e93d38b94d396b 2024-12-05T07:41:33,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/7b7d060e22f645dcb8e93d38b94d396b, entries=200, sequenceid=78, filesize=38.6 K 2024-12-05T07:41:33,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/9fc91ecee8824c2695d228c11bd2fda1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/9fc91ecee8824c2695d228c11bd2fda1 2024-12-05T07:41:33,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/9fc91ecee8824c2695d228c11bd2fda1, entries=150, sequenceid=78, filesize=11.7 K 2024-12-05T07:41:33,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/87b74ce7afe54c688ff07f7a58d3de75 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/87b74ce7afe54c688ff07f7a58d3de75 2024-12-05T07:41:33,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/87b74ce7afe54c688ff07f7a58d3de75, entries=150, sequenceid=78, filesize=11.7 K 2024-12-05T07:41:33,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2d9b639d709a2e4f040a3e7eadeae633 in 1263ms, sequenceid=78, compaction requested=false 2024-12-05T07:41:33,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:33,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T07:41:33,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:33,975 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:41:33,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057a44a62d455f47289ae330b739408e9a_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384492708/Put/seqid=0 2024-12-05T07:41:33,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742153_1329 (size=12154) 2024-12-05T07:41:34,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:34,391 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057a44a62d455f47289ae330b739408e9a_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057a44a62d455f47289ae330b739408e9a_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:34,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f7538354a54d4c99acadb1c85bed146d, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:34,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f7538354a54d4c99acadb1c85bed146d is 175, key is test_row_0/A:col10/1733384492708/Put/seqid=0 2024-12-05T07:41:34,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742154_1330 (size=30955) 2024-12-05T07:41:34,396 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f7538354a54d4c99acadb1c85bed146d 2024-12-05T07:41:34,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/c5129ce2ba1f44a58f6a5132e99e5c31 is 50, key is test_row_0/B:col10/1733384492708/Put/seqid=0 2024-12-05T07:41:34,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742155_1331 (size=12001) 2024-12-05T07:41:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T07:41:34,812 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/c5129ce2ba1f44a58f6a5132e99e5c31 2024-12-05T07:41:34,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/d55ae993a98d440d988eaa91bac06633 is 50, key is test_row_0/C:col10/1733384492708/Put/seqid=0 2024-12-05T07:41:34,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742156_1332 (size=12001) 2024-12-05T07:41:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:34,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384554929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:34,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384554930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:34,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384555045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384555046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,222 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/d55ae993a98d440d988eaa91bac06633 2024-12-05T07:41:35,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f7538354a54d4c99acadb1c85bed146d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f7538354a54d4c99acadb1c85bed146d 2024-12-05T07:41:35,228 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f7538354a54d4c99acadb1c85bed146d, entries=150, sequenceid=92, filesize=30.2 K 2024-12-05T07:41:35,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/c5129ce2ba1f44a58f6a5132e99e5c31 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/c5129ce2ba1f44a58f6a5132e99e5c31 2024-12-05T07:41:35,233 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/c5129ce2ba1f44a58f6a5132e99e5c31, entries=150, sequenceid=92, filesize=11.7 K 2024-12-05T07:41:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/d55ae993a98d440d988eaa91bac06633 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/d55ae993a98d440d988eaa91bac06633 2024-12-05T07:41:35,236 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/d55ae993a98d440d988eaa91bac06633, entries=150, sequenceid=92, filesize=11.7 K 2024-12-05T07:41:35,237 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2d9b639d709a2e4f040a3e7eadeae633 in 1262ms, sequenceid=92, compaction requested=true 2024-12-05T07:41:35,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:35,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:35,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-05T07:41:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-05T07:41:35,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-05T07:41:35,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6380 sec 2024-12-05T07:41:35,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.6430 sec 2024-12-05T07:41:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:35,261 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:35,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T07:41:35,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120522ee2d66a30e4335be87698085d77344_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384495259/Put/seqid=0 2024-12-05T07:41:35,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742157_1333 (size=17034) 2024-12-05T07:41:35,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:35,279 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120522ee2d66a30e4335be87698085d77344_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120522ee2d66a30e4335be87698085d77344_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:35,280 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/73e17bf52f6d4cd7bb945976ea4ead82, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:35,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/73e17bf52f6d4cd7bb945976ea4ead82 is 175, key is test_row_0/A:col10/1733384495259/Put/seqid=0 2024-12-05T07:41:35,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742158_1334 (size=48139) 2024-12-05T07:41:35,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384555285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384555286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384555395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:35,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384555406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384555526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:35,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:35,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384555526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:35,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:35,529 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 2d9b639d709a2e4f040a3e7eadeae633 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:35,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384555526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:35,530 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 2d9b639d709a2e4f040a3e7eadeae633 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,530 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 2d9b639d709a2e4f040a3e7eadeae633 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384555606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384555616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:35,684 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/73e17bf52f6d4cd7bb945976ea4ead82 2024-12-05T07:41:35,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/73a6c57947654555977a1698c047cfd1 is 50, key is test_row_0/B:col10/1733384495259/Put/seqid=0 2024-12-05T07:41:35,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742159_1335 (size=12001) 2024-12-05T07:41:35,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/73a6c57947654555977a1698c047cfd1 2024-12-05T07:41:35,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/356f1065a33b4ded8f5692dea7ea2f17 is 50, key is test_row_0/C:col10/1733384495259/Put/seqid=0 2024-12-05T07:41:35,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742160_1336 (size=12001) 2024-12-05T07:41:35,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/356f1065a33b4ded8f5692dea7ea2f17 2024-12-05T07:41:35,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/73e17bf52f6d4cd7bb945976ea4ead82 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/73e17bf52f6d4cd7bb945976ea4ead82 2024-12-05T07:41:35,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/73e17bf52f6d4cd7bb945976ea4ead82, entries=250, sequenceid=119, filesize=47.0 K 2024-12-05T07:41:35,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/73a6c57947654555977a1698c047cfd1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/73a6c57947654555977a1698c047cfd1 2024-12-05T07:41:35,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/73a6c57947654555977a1698c047cfd1, entries=150, sequenceid=119, filesize=11.7 K 2024-12-05T07:41:35,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/356f1065a33b4ded8f5692dea7ea2f17 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/356f1065a33b4ded8f5692dea7ea2f17 2024-12-05T07:41:35,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/356f1065a33b4ded8f5692dea7ea2f17, entries=150, sequenceid=119, filesize=11.7 K 2024-12-05T07:41:35,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 2d9b639d709a2e4f040a3e7eadeae633 in 467ms, sequenceid=119, compaction requested=true 2024-12-05T07:41:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:35,728 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:41:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:35,728 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:41:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:35,729 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:41:35,729 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:41:35,730 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/A is initiating minor compaction (all files) 2024-12-05T07:41:35,730 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/B is initiating minor compaction (all files) 2024-12-05T07:41:35,730 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/A in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:35,730 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/B in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:35,730 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f83590927ee349ba92fff1429ad0f5f2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/9fc91ecee8824c2695d228c11bd2fda1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/c5129ce2ba1f44a58f6a5132e99e5c31, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/73a6c57947654555977a1698c047cfd1] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=47.0 K 2024-12-05T07:41:35,730 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/c0c5d14d36344acdb997d18467df9b57, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/7b7d060e22f645dcb8e93d38b94d396b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f7538354a54d4c99acadb1c85bed146d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/73e17bf52f6d4cd7bb945976ea4ead82] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=146.2 K 2024-12-05T07:41:35,730 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:35,730 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/c0c5d14d36344acdb997d18467df9b57, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/7b7d060e22f645dcb8e93d38b94d396b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f7538354a54d4c99acadb1c85bed146d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/73e17bf52f6d4cd7bb945976ea4ead82] 2024-12-05T07:41:35,730 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f83590927ee349ba92fff1429ad0f5f2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733384489130 2024-12-05T07:41:35,730 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fc91ecee8824c2695d228c11bd2fda1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733384491319 2024-12-05T07:41:35,730 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0c5d14d36344acdb997d18467df9b57, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733384489130 2024-12-05T07:41:35,731 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b7d060e22f645dcb8e93d38b94d396b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733384491319 2024-12-05T07:41:35,731 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting c5129ce2ba1f44a58f6a5132e99e5c31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733384492708 2024-12-05T07:41:35,731 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7538354a54d4c99acadb1c85bed146d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733384492708 2024-12-05T07:41:35,731 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 73a6c57947654555977a1698c047cfd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733384494920 2024-12-05T07:41:35,731 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73e17bf52f6d4cd7bb945976ea4ead82, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733384494920 2024-12-05T07:41:35,739 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:35,739 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#B#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:35,740 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/16924b960c4d486fbdf79f7638cea0de is 50, key is test_row_0/B:col10/1733384495259/Put/seqid=0 2024-12-05T07:41:35,741 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412050dcaaee22d5e4d5d8b7486107ee0c2c2_2d9b639d709a2e4f040a3e7eadeae633 store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:35,742 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412050dcaaee22d5e4d5d8b7486107ee0c2c2_2d9b639d709a2e4f040a3e7eadeae633, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:35,742 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050dcaaee22d5e4d5d8b7486107ee0c2c2_2d9b639d709a2e4f040a3e7eadeae633 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:35,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742161_1337 (size=12241) 2024-12-05T07:41:35,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742162_1338 (size=4469) 2024-12-05T07:41:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:35,925 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:35,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:41:35,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053a7bc52ab26b446a870819fd96a8f592_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384495918/Put/seqid=0 2024-12-05T07:41:35,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742163_1339 (size=14694) 2024-12-05T07:41:36,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384556004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:36,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384556006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:36,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,016 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:36,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384556116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:36,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384556126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:36,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:36,154 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#A#compaction#285 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:36,154 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/1faefe7390594b32ae72d6b54546e710 is 175, key is test_row_0/A:col10/1733384495259/Put/seqid=0 2024-12-05T07:41:36,158 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/16924b960c4d486fbdf79f7638cea0de as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/16924b960c4d486fbdf79f7638cea0de 2024-12-05T07:41:36,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742164_1340 (size=31195) 2024-12-05T07:41:36,162 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/B of 2d9b639d709a2e4f040a3e7eadeae633 into 16924b960c4d486fbdf79f7638cea0de(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:36,162 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:36,162 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/B, priority=12, startTime=1733384495728; duration=0sec 2024-12-05T07:41:36,162 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:36,162 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:B 2024-12-05T07:41:36,162 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:41:36,163 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:41:36,163 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/C is initiating minor compaction (all files) 2024-12-05T07:41:36,163 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/C in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:36,163 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/43eb62f7a0074a079b17730bff9ddc9d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/87b74ce7afe54c688ff07f7a58d3de75, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/d55ae993a98d440d988eaa91bac06633, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/356f1065a33b4ded8f5692dea7ea2f17] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=47.0 K 2024-12-05T07:41:36,163 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 43eb62f7a0074a079b17730bff9ddc9d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1733384489130 2024-12-05T07:41:36,164 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 87b74ce7afe54c688ff07f7a58d3de75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733384491319 2024-12-05T07:41:36,164 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting d55ae993a98d440d988eaa91bac06633, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1733384492708 2024-12-05T07:41:36,164 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 356f1065a33b4ded8f5692dea7ea2f17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733384494920 2024-12-05T07:41:36,171 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#C#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:36,172 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/f34e3e6158cb4157ae59461c53ab7918 is 50, key is test_row_0/C:col10/1733384495259/Put/seqid=0 2024-12-05T07:41:36,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742165_1341 (size=12241) 2024-12-05T07:41:36,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384556326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:36,333 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:36,337 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053a7bc52ab26b446a870819fd96a8f592_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053a7bc52ab26b446a870819fd96a8f592_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:36,337 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/4ae9371e74a04414ac10340a5199bda9, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:36,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384556336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:36,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/4ae9371e74a04414ac10340a5199bda9 is 175, key is test_row_0/A:col10/1733384495918/Put/seqid=0 2024-12-05T07:41:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:36,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742166_1342 (size=39649) 2024-12-05T07:41:36,564 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/1faefe7390594b32ae72d6b54546e710 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1faefe7390594b32ae72d6b54546e710 2024-12-05T07:41:36,568 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/A of 2d9b639d709a2e4f040a3e7eadeae633 into 1faefe7390594b32ae72d6b54546e710(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:36,568 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:36,568 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/A, priority=12, startTime=1733384495728; duration=0sec 2024-12-05T07:41:36,568 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:36,568 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:A 2024-12-05T07:41:36,584 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/f34e3e6158cb4157ae59461c53ab7918 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f34e3e6158cb4157ae59461c53ab7918 2024-12-05T07:41:36,589 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/C of 2d9b639d709a2e4f040a3e7eadeae633 into f34e3e6158cb4157ae59461c53ab7918(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:36,589 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:36,589 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/C, priority=12, startTime=1733384495728; duration=0sec 2024-12-05T07:41:36,589 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:36,589 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:C 2024-12-05T07:41:36,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384556636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:36,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384556645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:36,746 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/4ae9371e74a04414ac10340a5199bda9 2024-12-05T07:41:36,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/f54b2b6c0b1f4651800b8802898d9b35 is 50, key is test_row_0/B:col10/1733384495918/Put/seqid=0 2024-12-05T07:41:36,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742167_1343 (size=12101) 2024-12-05T07:41:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T07:41:36,776 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:36,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-05T07:41:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T07:41:36,778 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:36,779 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:36,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T07:41:36,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T07:41:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:36,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T07:41:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:37,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:37,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T07:41:37,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:37,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384557146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:37,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/f54b2b6c0b1f4651800b8802898d9b35 2024-12-05T07:41:37,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/fc7ef22525494f4c82d101561eb183a7 is 50, key is test_row_0/C:col10/1733384495918/Put/seqid=0 2024-12-05T07:41:37,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:37,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384557156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:37,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742168_1344 (size=12101) 2024-12-05T07:41:37,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T07:41:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:37,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T07:41:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:37,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T07:41:37,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T07:41:37,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:37,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:37,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:37,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:37,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/fc7ef22525494f4c82d101561eb183a7 2024-12-05T07:41:37,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/4ae9371e74a04414ac10340a5199bda9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/4ae9371e74a04414ac10340a5199bda9 2024-12-05T07:41:37,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/4ae9371e74a04414ac10340a5199bda9, entries=200, sequenceid=130, filesize=38.7 K 2024-12-05T07:41:37,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/f54b2b6c0b1f4651800b8802898d9b35 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f54b2b6c0b1f4651800b8802898d9b35 2024-12-05T07:41:37,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f54b2b6c0b1f4651800b8802898d9b35, entries=150, sequenceid=130, filesize=11.8 K 2024-12-05T07:41:37,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/fc7ef22525494f4c82d101561eb183a7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/fc7ef22525494f4c82d101561eb183a7 2024-12-05T07:41:37,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/fc7ef22525494f4c82d101561eb183a7, entries=150, sequenceid=130, filesize=11.8 K 2024-12-05T07:41:37,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2d9b639d709a2e4f040a3e7eadeae633 in 1667ms, sequenceid=130, compaction requested=false 2024-12-05T07:41:37,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:37,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T07:41:37,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:37,694 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T07:41:37,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120529b16b44a3c3429aa4e6d49014b7119f_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384496003/Put/seqid=0 2024-12-05T07:41:37,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742169_1345 (size=12304) 2024-12-05T07:41:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T07:41:38,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:38,108 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120529b16b44a3c3429aa4e6d49014b7119f_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120529b16b44a3c3429aa4e6d49014b7119f_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:38,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/b84e6227ee5347488cdb816e185b6013, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:38,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/b84e6227ee5347488cdb816e185b6013 is 175, key is test_row_0/A:col10/1733384496003/Put/seqid=0 2024-12-05T07:41:38,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742170_1346 (size=31105) 2024-12-05T07:41:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:38,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:38,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384558189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:38,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384558192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:38,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:38,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384558296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:38,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:38,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384558306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:38,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384558506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:38,512 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/b84e6227ee5347488cdb816e185b6013 2024-12-05T07:41:38,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/f1e3f8e5e3f44c24bbdb1de5e523222b is 50, key is test_row_0/B:col10/1733384496003/Put/seqid=0 2024-12-05T07:41:38,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384558515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:38,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742171_1347 (size=12151) 2024-12-05T07:41:38,524 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/f1e3f8e5e3f44c24bbdb1de5e523222b 2024-12-05T07:41:38,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/3d9a9dba803143ccb36c682ed8c553a5 is 50, key is test_row_0/C:col10/1733384496003/Put/seqid=0 2024-12-05T07:41:38,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742172_1348 (size=12151) 2024-12-05T07:41:38,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:38,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384558816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:38,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:38,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384558826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T07:41:38,937 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/3d9a9dba803143ccb36c682ed8c553a5 2024-12-05T07:41:38,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/b84e6227ee5347488cdb816e185b6013 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/b84e6227ee5347488cdb816e185b6013 2024-12-05T07:41:38,944 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/b84e6227ee5347488cdb816e185b6013, entries=150, sequenceid=158, filesize=30.4 K 2024-12-05T07:41:38,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/f1e3f8e5e3f44c24bbdb1de5e523222b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f1e3f8e5e3f44c24bbdb1de5e523222b 2024-12-05T07:41:38,947 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f1e3f8e5e3f44c24bbdb1de5e523222b, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T07:41:38,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/3d9a9dba803143ccb36c682ed8c553a5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3d9a9dba803143ccb36c682ed8c553a5 2024-12-05T07:41:38,950 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3d9a9dba803143ccb36c682ed8c553a5, entries=150, sequenceid=158, filesize=11.9 K 2024-12-05T07:41:38,951 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2d9b639d709a2e4f040a3e7eadeae633 in 1257ms, sequenceid=158, compaction requested=true 2024-12-05T07:41:38,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:38,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:38,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-05T07:41:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-05T07:41:38,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-05T07:41:38,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1730 sec 2024-12-05T07:41:38,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.1780 sec 2024-12-05T07:41:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:39,330 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:39,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T07:41:39,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c41f8b850e4846ba985a5c19af8177eb_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384498181/Put/seqid=0 2024-12-05T07:41:39,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742173_1349 (size=14794) 2024-12-05T07:41:39,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384559400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:39,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384559405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384559515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384559515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:39,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54822 deadline: 1733384559546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:39,550 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 2d9b639d709a2e4f040a3e7eadeae633 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:39,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54818 deadline: 1733384559575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:39,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54814 deadline: 1733384559575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:39,581 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 2d9b639d709a2e4f040a3e7eadeae633 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,581 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 2d9b639d709a2e4f040a3e7eadeae633 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:41:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:39,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384559726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:39,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:39,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384559726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:39,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:39,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:39,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:39,770 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:39,773 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c41f8b850e4846ba985a5c19af8177eb_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c41f8b850e4846ba985a5c19af8177eb_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:39,774 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/410bf201bad3468d93afac5419986247, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:39,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/410bf201bad3468d93afac5419986247 is 175, key is test_row_0/A:col10/1733384498181/Put/seqid=0 2024-12-05T07:41:39,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742174_1350 (size=39749) 2024-12-05T07:41:39,778 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/410bf201bad3468d93afac5419986247 2024-12-05T07:41:39,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/d0d129fa1937453ba8e762befc38e3f7 is 50, key is test_row_0/B:col10/1733384498181/Put/seqid=0 2024-12-05T07:41:39,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742175_1351 (size=12151) 2024-12-05T07:41:40,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384560036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:40,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:40,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384560036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:40,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/d0d129fa1937453ba8e762befc38e3f7 2024-12-05T07:41:40,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/eed992edb82a4ef39f04cabf10c4c030 is 50, key is test_row_0/C:col10/1733384498181/Put/seqid=0 2024-12-05T07:41:40,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742176_1352 (size=12151) 2024-12-05T07:41:40,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384560546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:40,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:40,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:40,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384560556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:40,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:40,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:40,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:40,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/eed992edb82a4ef39f04cabf10c4c030 2024-12-05T07:41:40,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/410bf201bad3468d93afac5419986247 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/410bf201bad3468d93afac5419986247 2024-12-05T07:41:40,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/410bf201bad3468d93afac5419986247, entries=200, sequenceid=170, filesize=38.8 K 2024-12-05T07:41:40,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/d0d129fa1937453ba8e762befc38e3f7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/d0d129fa1937453ba8e762befc38e3f7 2024-12-05T07:41:40,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/d0d129fa1937453ba8e762befc38e3f7, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T07:41:40,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/eed992edb82a4ef39f04cabf10c4c030 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/eed992edb82a4ef39f04cabf10c4c030 2024-12-05T07:41:40,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/eed992edb82a4ef39f04cabf10c4c030, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T07:41:40,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2d9b639d709a2e4f040a3e7eadeae633 in 1283ms, sequenceid=170, compaction requested=true 2024-12-05T07:41:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:40,613 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:41:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:40,613 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:41:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:40,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:40,614 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:41:40,614 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:41:40,614 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/A is initiating minor compaction (all files) 2024-12-05T07:41:40,614 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/B is initiating minor compaction (all files) 2024-12-05T07:41:40,614 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/A in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:40,614 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/B in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:40,614 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1faefe7390594b32ae72d6b54546e710, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/4ae9371e74a04414ac10340a5199bda9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/b84e6227ee5347488cdb816e185b6013, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/410bf201bad3468d93afac5419986247] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=138.4 K 2024-12-05T07:41:40,614 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/16924b960c4d486fbdf79f7638cea0de, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f54b2b6c0b1f4651800b8802898d9b35, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f1e3f8e5e3f44c24bbdb1de5e523222b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/d0d129fa1937453ba8e762befc38e3f7] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=47.5 K 2024-12-05T07:41:40,614 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:40,614 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1faefe7390594b32ae72d6b54546e710, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/4ae9371e74a04414ac10340a5199bda9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/b84e6227ee5347488cdb816e185b6013, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/410bf201bad3468d93afac5419986247] 2024-12-05T07:41:40,614 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 16924b960c4d486fbdf79f7638cea0de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733384494920 2024-12-05T07:41:40,614 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1faefe7390594b32ae72d6b54546e710, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733384494920 2024-12-05T07:41:40,615 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f54b2b6c0b1f4651800b8802898d9b35, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733384495283 2024-12-05T07:41:40,615 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae9371e74a04414ac10340a5199bda9, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733384495283 2024-12-05T07:41:40,615 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f1e3f8e5e3f44c24bbdb1de5e523222b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733384495988 2024-12-05T07:41:40,615 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b84e6227ee5347488cdb816e185b6013, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733384495988 2024-12-05T07:41:40,615 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting d0d129fa1937453ba8e762befc38e3f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733384498181 2024-12-05T07:41:40,615 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 410bf201bad3468d93afac5419986247, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733384498177 2024-12-05T07:41:40,622 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:40,623 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#B#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:40,624 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/5b7477658598455bb81f534c40f5af1e is 50, key is test_row_0/B:col10/1733384498181/Put/seqid=0 2024-12-05T07:41:40,635 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412056423fdbf69ef419bb00699d16fddd6f6_2d9b639d709a2e4f040a3e7eadeae633 store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:40,636 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412056423fdbf69ef419bb00699d16fddd6f6_2d9b639d709a2e4f040a3e7eadeae633, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:40,636 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056423fdbf69ef419bb00699d16fddd6f6_2d9b639d709a2e4f040a3e7eadeae633 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:40,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742177_1353 (size=12527) 2024-12-05T07:41:40,644 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/5b7477658598455bb81f534c40f5af1e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/5b7477658598455bb81f534c40f5af1e 2024-12-05T07:41:40,649 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/B of 2d9b639d709a2e4f040a3e7eadeae633 into 5b7477658598455bb81f534c40f5af1e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:40,649 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:40,649 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/B, priority=12, startTime=1733384500613; duration=0sec 2024-12-05T07:41:40,649 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:40,649 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:B 2024-12-05T07:41:40,649 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:41:40,650 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:41:40,650 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/C is initiating minor compaction (all files) 2024-12-05T07:41:40,650 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/C in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:40,650 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f34e3e6158cb4157ae59461c53ab7918, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/fc7ef22525494f4c82d101561eb183a7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3d9a9dba803143ccb36c682ed8c553a5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/eed992edb82a4ef39f04cabf10c4c030] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=47.5 K 2024-12-05T07:41:40,650 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f34e3e6158cb4157ae59461c53ab7918, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733384494920 2024-12-05T07:41:40,650 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting fc7ef22525494f4c82d101561eb183a7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733384495283 2024-12-05T07:41:40,650 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d9a9dba803143ccb36c682ed8c553a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733384495988 2024-12-05T07:41:40,651 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting eed992edb82a4ef39f04cabf10c4c030, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733384498181 2024-12-05T07:41:40,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742178_1354 (size=4469) 2024-12-05T07:41:40,654 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#A#compaction#297 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:40,655 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f3d5f9990df84633aa39be2f8e568ea3 is 175, key is test_row_0/A:col10/1733384498181/Put/seqid=0 2024-12-05T07:41:40,658 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#C#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:40,659 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/6944bc3e4d20405187bbf9879925d5df is 50, key is test_row_0/C:col10/1733384498181/Put/seqid=0 2024-12-05T07:41:40,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742179_1355 (size=31481) 2024-12-05T07:41:40,665 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f3d5f9990df84633aa39be2f8e568ea3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3d5f9990df84633aa39be2f8e568ea3 2024-12-05T07:41:40,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742180_1356 (size=12527) 2024-12-05T07:41:40,670 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/A of 2d9b639d709a2e4f040a3e7eadeae633 into f3d5f9990df84633aa39be2f8e568ea3(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:40,670 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:40,670 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/A, priority=12, startTime=1733384500613; duration=0sec 2024-12-05T07:41:40,670 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:40,670 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:A 2024-12-05T07:41:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T07:41:40,956 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:40,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:40,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-05T07:41:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T07:41:40,958 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:40,959 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:40,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T07:41:41,073 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/6944bc3e4d20405187bbf9879925d5df as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/6944bc3e4d20405187bbf9879925d5df 2024-12-05T07:41:41,077 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/C of 2d9b639d709a2e4f040a3e7eadeae633 into 6944bc3e4d20405187bbf9879925d5df(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:41,077 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:41,077 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/C, priority=12, startTime=1733384500613; duration=0sec 2024-12-05T07:41:41,077 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:41,077 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:C 2024-12-05T07:41:41,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T07:41:41,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:41,111 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T07:41:41,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059b191ed0bf42487fa4f5825ee13b125b_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384499399/Put/seqid=0 2024-12-05T07:41:41,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742181_1357 (size=12304) 2024-12-05T07:41:41,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T07:41:41,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,525 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059b191ed0bf42487fa4f5825ee13b125b_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059b191ed0bf42487fa4f5825ee13b125b_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:41,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/cef59acae12e4374945e6af5778e1084, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:41,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/cef59acae12e4374945e6af5778e1084 is 175, key is test_row_0/A:col10/1733384499399/Put/seqid=0 2024-12-05T07:41:41,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742182_1358 (size=31105) 2024-12-05T07:41:41,529 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/cef59acae12e4374945e6af5778e1084 2024-12-05T07:41:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/50dcadefdb81466b99865181f7a46b84 is 50, key is test_row_0/B:col10/1733384499399/Put/seqid=0 2024-12-05T07:41:41,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742183_1359 (size=12151) 2024-12-05T07:41:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:41,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T07:41:41,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384561583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:41,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384561588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384561695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384561695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:41,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384561906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:41,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384561906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:41,942 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/50dcadefdb81466b99865181f7a46b84 2024-12-05T07:41:41,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/7b7704b5ea0d41bbb7fdf3b47e23b569 is 50, key is test_row_0/C:col10/1733384499399/Put/seqid=0 2024-12-05T07:41:41,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742184_1360 (size=12151) 2024-12-05T07:41:41,956 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/7b7704b5ea0d41bbb7fdf3b47e23b569 2024-12-05T07:41:41,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/cef59acae12e4374945e6af5778e1084 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/cef59acae12e4374945e6af5778e1084 2024-12-05T07:41:41,966 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/cef59acae12e4374945e6af5778e1084, entries=150, sequenceid=197, filesize=30.4 K 2024-12-05T07:41:41,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/50dcadefdb81466b99865181f7a46b84 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/50dcadefdb81466b99865181f7a46b84 2024-12-05T07:41:41,969 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/50dcadefdb81466b99865181f7a46b84, entries=150, sequenceid=197, filesize=11.9 K 2024-12-05T07:41:41,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/7b7704b5ea0d41bbb7fdf3b47e23b569 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/7b7704b5ea0d41bbb7fdf3b47e23b569 2024-12-05T07:41:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,973 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/7b7704b5ea0d41bbb7fdf3b47e23b569, entries=150, sequenceid=197, filesize=11.9 K 2024-12-05T07:41:41,973 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2d9b639d709a2e4f040a3e7eadeae633 in 862ms, sequenceid=197, compaction requested=false 2024-12-05T07:41:41,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:41,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:41,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-05T07:41:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-05T07:41:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-05T07:41:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0160 sec 2024-12-05T07:41:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.0210 sec 2024-12-05T07:41:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T07:41:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,096 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-05T07:41:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,099 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T07:41:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,100 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:42,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T07:41:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:42,227 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:42,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T07:41:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051bc8fd0b8e824a89955fc657dbe767a5_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384501586/Put/seqid=0 2024-12-05T07:41:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742185_1361 (size=14794) 2024-12-05T07:41:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:42,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384562323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:42,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:42,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:42,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384562324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:42,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:42,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T07:41:42,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384562436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:42,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:42,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384562436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:42,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:42,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:42,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,638 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:42,642 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051bc8fd0b8e824a89955fc657dbe767a5_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051bc8fd0b8e824a89955fc657dbe767a5_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:42,642 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/1b8cfca0e1f842c8a1a74d50a546df9e, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:42,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/1b8cfca0e1f842c8a1a74d50a546df9e is 175, key is test_row_0/A:col10/1733384501586/Put/seqid=0 2024-12-05T07:41:42,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742186_1362 (size=39749) 2024-12-05T07:41:42,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384562646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:42,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384562646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:42,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:42,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T07:41:42,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:42,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:42,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384562955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:42,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384562955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:42,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:43,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:43,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:43,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,046 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/1b8cfca0e1f842c8a1a74d50a546df9e 2024-12-05T07:41:43,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/7a0f3d89d26e4d1098dc95020bd97f8e is 50, key is test_row_0/B:col10/1733384501586/Put/seqid=0 2024-12-05T07:41:43,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742187_1363 (size=12151) 2024-12-05T07:41:43,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:43,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:43,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T07:41:43,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:43,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:43,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/7a0f3d89d26e4d1098dc95020bd97f8e 2024-12-05T07:41:43,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/f930584cb7c64e288eb125c9db788ef9 is 50, key is test_row_0/C:col10/1733384501586/Put/seqid=0 2024-12-05T07:41:43,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742188_1364 (size=12151) 2024-12-05T07:41:43,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384563465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:43,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384563466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:43,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:43,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:43,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:43,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:43,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/f930584cb7c64e288eb125c9db788ef9 2024-12-05T07:41:43,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/1b8cfca0e1f842c8a1a74d50a546df9e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1b8cfca0e1f842c8a1a74d50a546df9e 2024-12-05T07:41:43,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1b8cfca0e1f842c8a1a74d50a546df9e, entries=200, sequenceid=211, filesize=38.8 K 2024-12-05T07:41:43,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/7a0f3d89d26e4d1098dc95020bd97f8e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/7a0f3d89d26e4d1098dc95020bd97f8e 2024-12-05T07:41:43,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/7a0f3d89d26e4d1098dc95020bd97f8e, entries=150, sequenceid=211, filesize=11.9 K 2024-12-05T07:41:43,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/f930584cb7c64e288eb125c9db788ef9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f930584cb7c64e288eb125c9db788ef9 2024-12-05T07:41:43,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f930584cb7c64e288eb125c9db788ef9, entries=150, sequenceid=211, filesize=11.9 K 2024-12-05T07:41:43,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2d9b639d709a2e4f040a3e7eadeae633 in 1655ms, sequenceid=211, compaction requested=true 2024-12-05T07:41:43,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:43,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:43,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:43,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:43,882 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:43,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:43,882 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:43,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:43,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:43,883 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:43,883 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:43,883 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/B is initiating minor compaction (all files) 2024-12-05T07:41:43,883 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/A is initiating minor compaction (all files) 2024-12-05T07:41:43,883 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/A in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,883 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/B in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,883 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/5b7477658598455bb81f534c40f5af1e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/50dcadefdb81466b99865181f7a46b84, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/7a0f3d89d26e4d1098dc95020bd97f8e] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=36.0 K 2024-12-05T07:41:43,883 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3d5f9990df84633aa39be2f8e568ea3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/cef59acae12e4374945e6af5778e1084, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1b8cfca0e1f842c8a1a74d50a546df9e] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=99.9 K 2024-12-05T07:41:43,883 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,884 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3d5f9990df84633aa39be2f8e568ea3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/cef59acae12e4374945e6af5778e1084, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1b8cfca0e1f842c8a1a74d50a546df9e] 2024-12-05T07:41:43,884 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b7477658598455bb81f534c40f5af1e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733384498181 2024-12-05T07:41:43,884 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3d5f9990df84633aa39be2f8e568ea3, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733384498181 2024-12-05T07:41:43,884 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 50dcadefdb81466b99865181f7a46b84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733384499382 2024-12-05T07:41:43,884 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef59acae12e4374945e6af5778e1084, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733384499382 2024-12-05T07:41:43,884 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a0f3d89d26e4d1098dc95020bd97f8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733384501586 2024-12-05T07:41:43,884 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b8cfca0e1f842c8a1a74d50a546df9e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733384501582 2024-12-05T07:41:43,889 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#B#compaction#305 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:43,890 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/35fe0fd4b5c54494b18115686b6d7331 is 50, key is test_row_0/B:col10/1733384501586/Put/seqid=0 2024-12-05T07:41:43,895 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:43,897 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120501056ae142fa453ab7f6abdbf1b875a6_2d9b639d709a2e4f040a3e7eadeae633 store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:43,898 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120501056ae142fa453ab7f6abdbf1b875a6_2d9b639d709a2e4f040a3e7eadeae633, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:43,898 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120501056ae142fa453ab7f6abdbf1b875a6_2d9b639d709a2e4f040a3e7eadeae633 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:43,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742189_1365 (size=12629) 2024-12-05T07:41:43,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742190_1366 (size=4469) 2024-12-05T07:41:43,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T07:41:43,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:43,934 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-05T07:41:43,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054d1c3fb8ef60431b90b510ecbcaca11e_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384502314/Put/seqid=0 2024-12-05T07:41:43,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742191_1367 (size=12304) 2024-12-05T07:41:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T07:41:44,302 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/35fe0fd4b5c54494b18115686b6d7331 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/35fe0fd4b5c54494b18115686b6d7331 2024-12-05T07:41:44,306 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/B of 2d9b639d709a2e4f040a3e7eadeae633 into 35fe0fd4b5c54494b18115686b6d7331(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:44,306 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:44,306 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/B, priority=13, startTime=1733384503882; duration=0sec 2024-12-05T07:41:44,306 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:44,306 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:B 2024-12-05T07:41:44,306 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:44,308 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:44,308 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/C is initiating minor compaction (all files) 2024-12-05T07:41:44,308 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/C in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:44,308 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/6944bc3e4d20405187bbf9879925d5df, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/7b7704b5ea0d41bbb7fdf3b47e23b569, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f930584cb7c64e288eb125c9db788ef9] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=36.0 K 2024-12-05T07:41:44,308 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6944bc3e4d20405187bbf9879925d5df, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733384498181 2024-12-05T07:41:44,308 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#A#compaction#306 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:44,308 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7704b5ea0d41bbb7fdf3b47e23b569, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733384499382 2024-12-05T07:41:44,308 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f930584cb7c64e288eb125c9db788ef9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733384501586 2024-12-05T07:41:44,309 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/82fb9bd1bc46412b968047793ddc5d08 is 175, key is test_row_0/A:col10/1733384501586/Put/seqid=0 2024-12-05T07:41:44,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742192_1368 (size=31583) 2024-12-05T07:41:44,314 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#C#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:44,314 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/2be155c48b534714ab45325b470a85d3 is 50, key is test_row_0/C:col10/1733384501586/Put/seqid=0 2024-12-05T07:41:44,318 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/82fb9bd1bc46412b968047793ddc5d08 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/82fb9bd1bc46412b968047793ddc5d08 2024-12-05T07:41:44,322 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/A of 2d9b639d709a2e4f040a3e7eadeae633 into 82fb9bd1bc46412b968047793ddc5d08(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:44,322 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:44,322 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/A, priority=13, startTime=1733384503882; duration=0sec 2024-12-05T07:41:44,322 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:44,322 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:A 2024-12-05T07:41:44,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742193_1369 (size=12629) 2024-12-05T07:41:44,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:44,346 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054d1c3fb8ef60431b90b510ecbcaca11e_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054d1c3fb8ef60431b90b510ecbcaca11e_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:44,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f0d38699b5d849f086b322db62f74d61, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:44,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f0d38699b5d849f086b322db62f74d61 is 175, key is test_row_0/A:col10/1733384502314/Put/seqid=0 2024-12-05T07:41:44,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742194_1370 (size=31105) 2024-12-05T07:41:44,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:44,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384564516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:44,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384564517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:44,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384564626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:44,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384564626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:44,724 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/2be155c48b534714ab45325b470a85d3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/2be155c48b534714ab45325b470a85d3 2024-12-05T07:41:44,727 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/C of 2d9b639d709a2e4f040a3e7eadeae633 into 2be155c48b534714ab45325b470a85d3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:44,727 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:44,727 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/C, priority=13, startTime=1733384503882; duration=0sec 2024-12-05T07:41:44,727 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:44,727 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:C 2024-12-05T07:41:44,772 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f0d38699b5d849f086b322db62f74d61 2024-12-05T07:41:44,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/35740b69b07441a8a0766d98c08c0ea7 is 50, key is test_row_0/B:col10/1733384502314/Put/seqid=0 2024-12-05T07:41:44,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742195_1371 (size=12151) 2024-12-05T07:41:44,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384564836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:44,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384564836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:45,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384565146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384565146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:45,185 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/35740b69b07441a8a0766d98c08c0ea7 2024-12-05T07:41:45,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/785dfedb64d44372823e3a3bfbbdd24a is 50, key is test_row_0/C:col10/1733384502314/Put/seqid=0 2024-12-05T07:41:45,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742196_1372 (size=12151) 2024-12-05T07:41:45,594 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/785dfedb64d44372823e3a3bfbbdd24a 2024-12-05T07:41:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f0d38699b5d849f086b322db62f74d61 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f0d38699b5d849f086b322db62f74d61 2024-12-05T07:41:45,600 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f0d38699b5d849f086b322db62f74d61, entries=150, sequenceid=233, filesize=30.4 K 2024-12-05T07:41:45,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/35740b69b07441a8a0766d98c08c0ea7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/35740b69b07441a8a0766d98c08c0ea7 2024-12-05T07:41:45,604 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/35740b69b07441a8a0766d98c08c0ea7, entries=150, sequenceid=233, filesize=11.9 K 2024-12-05T07:41:45,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/785dfedb64d44372823e3a3bfbbdd24a as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/785dfedb64d44372823e3a3bfbbdd24a 2024-12-05T07:41:45,609 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/785dfedb64d44372823e3a3bfbbdd24a, entries=150, sequenceid=233, filesize=11.9 K 2024-12-05T07:41:45,609 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2d9b639d709a2e4f040a3e7eadeae633 in 1675ms, sequenceid=233, compaction requested=false 2024-12-05T07:41:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-05T07:41:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-05T07:41:45,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-05T07:41:45,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5110 sec 2024-12-05T07:41:45,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.5170 sec 2024-12-05T07:41:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:45,659 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:45,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-05T07:41:45,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e3b1836cb35747a58422e46953f58b8e_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384504516/Put/seqid=0 2024-12-05T07:41:45,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742197_1373 (size=12304) 2024-12-05T07:41:45,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384565718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:45,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384565719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:45,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384565835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:45,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384565836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:45,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:46,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384566046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:46,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384566046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:46,082 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:46,085 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e3b1836cb35747a58422e46953f58b8e_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e3b1836cb35747a58422e46953f58b8e_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:46,085 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/8e08c62142a74012b90b9a6808af5b6e, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:46,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/8e08c62142a74012b90b9a6808af5b6e is 175, key is test_row_0/A:col10/1733384504516/Put/seqid=0 2024-12-05T07:41:46,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742198_1374 (size=31105) 2024-12-05T07:41:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T07:41:46,256 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:46,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-05T07:41:46,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T07:41:46,258 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:46,258 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:46,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:46,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384566355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:46,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384566356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T07:41:46,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T07:41:46,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:46,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,490 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/8e08c62142a74012b90b9a6808af5b6e 2024-12-05T07:41:46,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/8417c03ec5c34c55a2140b6a43078c41 is 50, key is test_row_0/B:col10/1733384504516/Put/seqid=0 2024-12-05T07:41:46,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742199_1375 (size=12151) 2024-12-05T07:41:46,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/8417c03ec5c34c55a2140b6a43078c41 2024-12-05T07:41:46,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/3b6f5aaf543f42dbab104451c639f659 is 50, key is test_row_0/C:col10/1733384504516/Put/seqid=0 2024-12-05T07:41:46,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742200_1376 (size=12151) 2024-12-05T07:41:46,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T07:41:46,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:46,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T07:41:46,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T07:41:46,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:46,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T07:41:46,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:46,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:46,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:46,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384566866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:46,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:46,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384566876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T07:41:46,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/3b6f5aaf543f42dbab104451c639f659 2024-12-05T07:41:46,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/8e08c62142a74012b90b9a6808af5b6e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8e08c62142a74012b90b9a6808af5b6e 2024-12-05T07:41:46,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8e08c62142a74012b90b9a6808af5b6e, entries=150, sequenceid=251, filesize=30.4 K 2024-12-05T07:41:46,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/8417c03ec5c34c55a2140b6a43078c41 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/8417c03ec5c34c55a2140b6a43078c41 2024-12-05T07:41:46,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/8417c03ec5c34c55a2140b6a43078c41, entries=150, sequenceid=251, filesize=11.9 K 2024-12-05T07:41:46,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/3b6f5aaf543f42dbab104451c639f659 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3b6f5aaf543f42dbab104451c639f659 2024-12-05T07:41:46,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3b6f5aaf543f42dbab104451c639f659, entries=150, sequenceid=251, filesize=11.9 K 2024-12-05T07:41:46,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 2d9b639d709a2e4f040a3e7eadeae633 in 1266ms, sequenceid=251, compaction requested=true 2024-12-05T07:41:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:46,926 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:46,926 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:46,926 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:46,926 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93793 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:46,927 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/B is initiating minor compaction (all files) 2024-12-05T07:41:46,927 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/A is initiating minor compaction (all files) 2024-12-05T07:41:46,927 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/B in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,927 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/A in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,927 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/35fe0fd4b5c54494b18115686b6d7331, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/35740b69b07441a8a0766d98c08c0ea7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/8417c03ec5c34c55a2140b6a43078c41] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=36.1 K 2024-12-05T07:41:46,927 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/82fb9bd1bc46412b968047793ddc5d08, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f0d38699b5d849f086b322db62f74d61, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8e08c62142a74012b90b9a6808af5b6e] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=91.6 K 2024-12-05T07:41:46,927 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,927 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/82fb9bd1bc46412b968047793ddc5d08, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f0d38699b5d849f086b322db62f74d61, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8e08c62142a74012b90b9a6808af5b6e] 2024-12-05T07:41:46,927 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 35fe0fd4b5c54494b18115686b6d7331, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733384501586 2024-12-05T07:41:46,927 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82fb9bd1bc46412b968047793ddc5d08, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733384501586 2024-12-05T07:41:46,927 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 35740b69b07441a8a0766d98c08c0ea7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733384502314 2024-12-05T07:41:46,927 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0d38699b5d849f086b322db62f74d61, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733384502314 2024-12-05T07:41:46,927 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8417c03ec5c34c55a2140b6a43078c41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733384504505 2024-12-05T07:41:46,927 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e08c62142a74012b90b9a6808af5b6e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733384504505 2024-12-05T07:41:46,931 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:46,933 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#B#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:46,933 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/58e597507cfa4d5fad13e593e2cb4d0c is 50, key is test_row_0/B:col10/1733384504516/Put/seqid=0 2024-12-05T07:41:46,935 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412053a98d038140b4e6c9c0ea39f8cb53103_2d9b639d709a2e4f040a3e7eadeae633 store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:46,937 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412053a98d038140b4e6c9c0ea39f8cb53103_2d9b639d709a2e4f040a3e7eadeae633, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:46,937 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053a98d038140b4e6c9c0ea39f8cb53103_2d9b639d709a2e4f040a3e7eadeae633 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:46,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742202_1378 (size=4469) 2024-12-05T07:41:46,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742201_1377 (size=12731) 2024-12-05T07:41:46,947 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/58e597507cfa4d5fad13e593e2cb4d0c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/58e597507cfa4d5fad13e593e2cb4d0c 2024-12-05T07:41:46,952 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/B of 2d9b639d709a2e4f040a3e7eadeae633 into 58e597507cfa4d5fad13e593e2cb4d0c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:46,952 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:46,952 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/B, priority=13, startTime=1733384506926; duration=0sec 2024-12-05T07:41:46,952 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:46,952 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:B 2024-12-05T07:41:46,952 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:46,953 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:46,953 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/C is initiating minor compaction (all files) 2024-12-05T07:41:46,953 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/C in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:46,953 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/2be155c48b534714ab45325b470a85d3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/785dfedb64d44372823e3a3bfbbdd24a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3b6f5aaf543f42dbab104451c639f659] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=36.1 K 2024-12-05T07:41:46,953 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2be155c48b534714ab45325b470a85d3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733384501586 2024-12-05T07:41:46,953 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 785dfedb64d44372823e3a3bfbbdd24a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733384502314 2024-12-05T07:41:46,954 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b6f5aaf543f42dbab104451c639f659, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733384504505 2024-12-05T07:41:46,960 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#C#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:46,960 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/f902204b278d41c6b767fde0a0aeb3cf is 50, key is test_row_0/C:col10/1733384504516/Put/seqid=0 2024-12-05T07:41:46,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742203_1379 (size=12731) 2024-12-05T07:41:47,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T07:41:47,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:47,021 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T07:41:47,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f969a21e5aa24008bf125e4b127c5438_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384505717/Put/seqid=0 2024-12-05T07:41:47,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742204_1380 (size=12454) 2024-12-05T07:41:47,341 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:47,342 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/ac8fc155a4444cd0a9f09a814e30af03 is 175, key is test_row_0/A:col10/1733384504516/Put/seqid=0 2024-12-05T07:41:47,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742205_1381 (size=31685) 2024-12-05T07:41:47,377 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/f902204b278d41c6b767fde0a0aeb3cf as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f902204b278d41c6b767fde0a0aeb3cf 2024-12-05T07:41:47,381 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/C of 2d9b639d709a2e4f040a3e7eadeae633 into f902204b278d41c6b767fde0a0aeb3cf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:47,381 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:47,381 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/C, priority=13, startTime=1733384506926; duration=0sec 2024-12-05T07:41:47,381 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:47,381 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:C 2024-12-05T07:41:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T07:41:47,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,436 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f969a21e5aa24008bf125e4b127c5438_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f969a21e5aa24008bf125e4b127c5438_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:47,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f3738cce963d40218703f903d1a967ca, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:47,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f3738cce963d40218703f903d1a967ca is 175, key is test_row_0/A:col10/1733384505717/Put/seqid=0 2024-12-05T07:41:47,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742206_1382 (size=31255) 2024-12-05T07:41:47,442 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f3738cce963d40218703f903d1a967ca 2024-12-05T07:41:47,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/6fd003a05d3a4aadb4b12cdebb93352e is 50, key is test_row_0/B:col10/1733384505717/Put/seqid=0 2024-12-05T07:41:47,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742207_1383 (size=12301) 2024-12-05T07:41:47,460 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/6fd003a05d3a4aadb4b12cdebb93352e 2024-12-05T07:41:47,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/79bb8b6911c9400a83505b8135e285e3 is 50, key is test_row_0/C:col10/1733384505717/Put/seqid=0 2024-12-05T07:41:47,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742208_1384 (size=12301) 2024-12-05T07:41:47,482 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/79bb8b6911c9400a83505b8135e285e3 2024-12-05T07:41:47,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/f3738cce963d40218703f903d1a967ca as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3738cce963d40218703f903d1a967ca 2024-12-05T07:41:47,491 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3738cce963d40218703f903d1a967ca, entries=150, sequenceid=273, filesize=30.5 K 2024-12-05T07:41:47,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/6fd003a05d3a4aadb4b12cdebb93352e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/6fd003a05d3a4aadb4b12cdebb93352e 2024-12-05T07:41:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,495 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/6fd003a05d3a4aadb4b12cdebb93352e, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T07:41:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/79bb8b6911c9400a83505b8135e285e3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/79bb8b6911c9400a83505b8135e285e3 2024-12-05T07:41:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,499 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/79bb8b6911c9400a83505b8135e285e3, entries=150, sequenceid=273, filesize=12.0 K 2024-12-05T07:41:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,500 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 2d9b639d709a2e4f040a3e7eadeae633 in 479ms, sequenceid=273, compaction requested=false 2024-12-05T07:41:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:47,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-05T07:41:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-05T07:41:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-05T07:41:47,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2430 sec 2024-12-05T07:41:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.2480 sec 2024-12-05T07:41:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,749 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/ac8fc155a4444cd0a9f09a814e30af03 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/ac8fc155a4444cd0a9f09a814e30af03 2024-12-05T07:41:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,758 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/A of 2d9b639d709a2e4f040a3e7eadeae633 into ac8fc155a4444cd0a9f09a814e30af03(size=30.9 K), total size for store is 61.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,758 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:47,758 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/A, priority=13, startTime=1733384506926; duration=0sec 2024-12-05T07:41:47,758 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,758 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:A 2024-12-05T07:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:48,068 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:48,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:41:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051b9bd6da7a704a42b7833432d9c07d06_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384508067/Put/seqid=0 2024-12-05T07:41:48,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742209_1385 (size=17534) 2024-12-05T07:41:48,095 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:48,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051b9bd6da7a704a42b7833432d9c07d06_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051b9bd6da7a704a42b7833432d9c07d06_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:48,101 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/d72506a3fa65497192b6a1a411475147, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:48,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/d72506a3fa65497192b6a1a411475147 is 175, key is test_row_0/A:col10/1733384508067/Put/seqid=0 2024-12-05T07:41:48,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742210_1386 (size=48639) 2024-12-05T07:41:48,117 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/d72506a3fa65497192b6a1a411475147 2024-12-05T07:41:48,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/e8a8f20a540f4b6aa9c3ade00c882d6b is 50, key is test_row_0/B:col10/1733384508067/Put/seqid=0 2024-12-05T07:41:48,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742211_1387 (size=12301) 2024-12-05T07:41:48,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384568170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:48,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:48,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384568180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384568286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:48,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384568296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T07:41:48,406 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:48,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-05T07:41:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T07:41:48,408 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:48,409 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:48,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:48,442 INFO [Thread-1430 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1430. 2024-12-05T07:41:48,442 DEBUG [Thread-1430 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:48,442 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:48,442 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:48,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:48,445 INFO [Thread-1434 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1434. 2024-12-05T07:41:48,445 DEBUG [Thread-1434 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:48,445 DEBUG [Thread-1434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:48,445 DEBUG [Thread-1434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:48,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:48,446 INFO [Thread-1438 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1438. 2024-12-05T07:41:48,446 DEBUG [Thread-1438 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:48,447 DEBUG [Thread-1438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:48,447 DEBUG [Thread-1438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:48,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:48,449 INFO [Thread-1436 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1436. 2024-12-05T07:41:48,449 DEBUG [Thread-1436 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:48,449 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:48,449 DEBUG [Thread-1436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:48,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:48,451 INFO [Thread-1432 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1432. 2024-12-05T07:41:48,451 DEBUG [Thread-1432 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:48,451 DEBUG [Thread-1432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:48,451 DEBUG [Thread-1432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:48,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384568496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:48,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384568506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T07:41:48,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/e8a8f20a540f4b6aa9c3ade00c882d6b 2024-12-05T07:41:48,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/4e0fd46ce32343c4bddf2a98d74b808b is 50, key is test_row_0/C:col10/1733384508067/Put/seqid=0 2024-12-05T07:41:48,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742212_1388 (size=12301) 2024-12-05T07:41:48,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T07:41:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:48,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:48,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:48,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T07:41:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:48,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T07:41:48,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54848 deadline: 1733384568806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:48,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54826 deadline: 1733384568816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:41:48,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T07:41:48,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:48,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:48,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:48,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:48,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:48,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:48,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/4e0fd46ce32343c4bddf2a98d74b808b 2024-12-05T07:41:48,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/d72506a3fa65497192b6a1a411475147 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/d72506a3fa65497192b6a1a411475147 2024-12-05T07:41:48,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/d72506a3fa65497192b6a1a411475147, entries=250, sequenceid=286, filesize=47.5 K 2024-12-05T07:41:48,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/e8a8f20a540f4b6aa9c3ade00c882d6b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/e8a8f20a540f4b6aa9c3ade00c882d6b 2024-12-05T07:41:48,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/e8a8f20a540f4b6aa9c3ade00c882d6b, entries=150, sequenceid=286, filesize=12.0 K 2024-12-05T07:41:48,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/4e0fd46ce32343c4bddf2a98d74b808b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/4e0fd46ce32343c4bddf2a98d74b808b 2024-12-05T07:41:48,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/4e0fd46ce32343c4bddf2a98d74b808b, entries=150, sequenceid=286, filesize=12.0 K 2024-12-05T07:41:48,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2d9b639d709a2e4f040a3e7eadeae633 in 910ms, sequenceid=286, compaction requested=true 2024-12-05T07:41:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:41:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:48,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:41:48,979 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:48,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:48,979 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:48,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d9b639d709a2e4f040a3e7eadeae633:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:41:48,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:48,979 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:48,979 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111579 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:48,979 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/B is initiating minor compaction (all files) 2024-12-05T07:41:48,979 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/A is initiating minor compaction (all files) 2024-12-05T07:41:48,980 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/B in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:48,980 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/A in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:48,980 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/58e597507cfa4d5fad13e593e2cb4d0c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/6fd003a05d3a4aadb4b12cdebb93352e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/e8a8f20a540f4b6aa9c3ade00c882d6b] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=36.5 K 2024-12-05T07:41:48,980 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/ac8fc155a4444cd0a9f09a814e30af03, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3738cce963d40218703f903d1a967ca, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/d72506a3fa65497192b6a1a411475147] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=109.0 K 2024-12-05T07:41:48,980 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:48,980 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/ac8fc155a4444cd0a9f09a814e30af03, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3738cce963d40218703f903d1a967ca, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/d72506a3fa65497192b6a1a411475147] 2024-12-05T07:41:48,980 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 58e597507cfa4d5fad13e593e2cb4d0c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733384504505 2024-12-05T07:41:48,980 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac8fc155a4444cd0a9f09a814e30af03, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733384504505 2024-12-05T07:41:48,980 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fd003a05d3a4aadb4b12cdebb93352e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733384505710 2024-12-05T07:41:48,980 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3738cce963d40218703f903d1a967ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733384505710 2024-12-05T07:41:48,980 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting e8a8f20a540f4b6aa9c3ade00c882d6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384508006 2024-12-05T07:41:48,981 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d72506a3fa65497192b6a1a411475147, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384508006 2024-12-05T07:41:48,985 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:48,986 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#B#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:48,986 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/c072f9a7966b4c3397229a59b0540c60 is 50, key is test_row_0/B:col10/1733384508067/Put/seqid=0 2024-12-05T07:41:48,987 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205870fa408a3ca41b69c0df00966d052b5_2d9b639d709a2e4f040a3e7eadeae633 store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:48,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742213_1389 (size=12983) 2024-12-05T07:41:48,990 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205870fa408a3ca41b69c0df00966d052b5_2d9b639d709a2e4f040a3e7eadeae633, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:48,990 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205870fa408a3ca41b69c0df00966d052b5_2d9b639d709a2e4f040a3e7eadeae633 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:48,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742214_1390 (size=4469) 2024-12-05T07:41:49,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T07:41:49,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:49,021 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T07:41:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120518c9ba38325c400cbafaae753815c026_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384508178/Put/seqid=0 2024-12-05T07:41:49,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742215_1391 (size=12454) 2024-12-05T07:41:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T07:41:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:49,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. as already flushing 2024-12-05T07:41:49,318 INFO [Thread-1427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1427. 2024-12-05T07:41:49,318 DEBUG [Thread-1427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:49,318 DEBUG [Thread-1427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:49,318 DEBUG [Thread-1427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:49,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:49,327 INFO [Thread-1425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1425. 2024-12-05T07:41:49,327 DEBUG [Thread-1425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:49,327 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:49,327 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:49,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:49,396 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#A#compaction#324 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:49,398 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/e4bda6f9fa504004b0b4b496d2676b09 is 175, key is test_row_0/A:col10/1733384508067/Put/seqid=0 2024-12-05T07:41:49,401 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/c072f9a7966b4c3397229a59b0540c60 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/c072f9a7966b4c3397229a59b0540c60 2024-12-05T07:41:49,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742216_1392 (size=31937) 2024-12-05T07:41:49,406 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/B of 2d9b639d709a2e4f040a3e7eadeae633 into c072f9a7966b4c3397229a59b0540c60(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:49,406 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:49,406 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/B, priority=13, startTime=1733384508978; duration=0sec 2024-12-05T07:41:49,406 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:41:49,406 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:B 2024-12-05T07:41:49,406 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:41:49,407 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:41:49,407 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 2d9b639d709a2e4f040a3e7eadeae633/C is initiating minor compaction (all files) 2024-12-05T07:41:49,407 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d9b639d709a2e4f040a3e7eadeae633/C in TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:49,407 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f902204b278d41c6b767fde0a0aeb3cf, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/79bb8b6911c9400a83505b8135e285e3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/4e0fd46ce32343c4bddf2a98d74b808b] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp, totalSize=36.5 K 2024-12-05T07:41:49,408 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f902204b278d41c6b767fde0a0aeb3cf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733384504505 2024-12-05T07:41:49,408 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 79bb8b6911c9400a83505b8135e285e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1733384505710 2024-12-05T07:41:49,408 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e0fd46ce32343c4bddf2a98d74b808b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384508006 2024-12-05T07:41:49,415 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d9b639d709a2e4f040a3e7eadeae633#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:41:49,415 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/ed413bf346304ac9a7f4e210d732e855 is 50, key is test_row_0/C:col10/1733384508067/Put/seqid=0 2024-12-05T07:41:49,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742217_1393 (size=12983) 2024-12-05T07:41:49,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:49,433 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120518c9ba38325c400cbafaae753815c026_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120518c9ba38325c400cbafaae753815c026_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:49,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/654ea0f2fcd14d62b0cd4e31618246cb, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:49,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/654ea0f2fcd14d62b0cd4e31618246cb is 175, key is test_row_0/A:col10/1733384508178/Put/seqid=0 2024-12-05T07:41:49,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742218_1394 (size=31255) 2024-12-05T07:41:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T07:41:49,640 INFO [Thread-1421 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1421. 2024-12-05T07:41:49,640 DEBUG [Thread-1421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:49,640 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:49,640 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:49,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:49,658 INFO [Thread-1423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1423. 2024-12-05T07:41:49,658 DEBUG [Thread-1423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:49,658 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:49,658 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:49,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:49,678 INFO [Thread-1419 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1419. 2024-12-05T07:41:49,678 DEBUG [Thread-1419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:41:49,678 DEBUG [Thread-1419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:49,679 DEBUG [Thread-1419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:49,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:49,855 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/654ea0f2fcd14d62b0cd4e31618246cb 2024-12-05T07:41:49,860 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/e4bda6f9fa504004b0b4b496d2676b09 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/e4bda6f9fa504004b0b4b496d2676b09 2024-12-05T07:41:49,864 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/ed413bf346304ac9a7f4e210d732e855 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/ed413bf346304ac9a7f4e210d732e855 2024-12-05T07:41:49,866 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/A of 2d9b639d709a2e4f040a3e7eadeae633 into e4bda6f9fa504004b0b4b496d2676b09(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:49,866 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:49,866 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/A, priority=13, startTime=1733384508978; duration=0sec 2024-12-05T07:41:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/fcac38ebf1a0447db6ba5a2066a0d36f is 50, key is test_row_0/B:col10/1733384508178/Put/seqid=0 2024-12-05T07:41:49,866 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:49,866 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:A 2024-12-05T07:41:49,868 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d9b639d709a2e4f040a3e7eadeae633/C of 2d9b639d709a2e4f040a3e7eadeae633 into ed413bf346304ac9a7f4e210d732e855(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:41:49,868 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:49,868 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., storeName=2d9b639d709a2e4f040a3e7eadeae633/C, priority=13, startTime=1733384508979; duration=0sec 2024-12-05T07:41:49,868 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:41:49,868 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d9b639d709a2e4f040a3e7eadeae633:C 2024-12-05T07:41:49,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742219_1395 (size=12301) 2024-12-05T07:41:50,272 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/fcac38ebf1a0447db6ba5a2066a0d36f 2024-12-05T07:41:50,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/bb37a08232b546f289650a0be7de0699 is 50, key is test_row_0/C:col10/1733384508178/Put/seqid=0 2024-12-05T07:41:50,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742220_1396 (size=12301) 2024-12-05T07:41:50,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T07:41:50,691 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/bb37a08232b546f289650a0be7de0699 2024-12-05T07:41:50,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/654ea0f2fcd14d62b0cd4e31618246cb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/654ea0f2fcd14d62b0cd4e31618246cb 2024-12-05T07:41:50,703 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/654ea0f2fcd14d62b0cd4e31618246cb, entries=150, sequenceid=311, filesize=30.5 K 2024-12-05T07:41:50,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/fcac38ebf1a0447db6ba5a2066a0d36f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/fcac38ebf1a0447db6ba5a2066a0d36f 2024-12-05T07:41:50,709 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/fcac38ebf1a0447db6ba5a2066a0d36f, entries=150, sequenceid=311, filesize=12.0 K 2024-12-05T07:41:50,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/bb37a08232b546f289650a0be7de0699 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/bb37a08232b546f289650a0be7de0699 2024-12-05T07:41:50,715 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/bb37a08232b546f289650a0be7de0699, entries=150, sequenceid=311, filesize=12.0 K 2024-12-05T07:41:50,716 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=33.54 KB/34350 for 2d9b639d709a2e4f040a3e7eadeae633 in 1695ms, sequenceid=311, compaction requested=false 2024-12-05T07:41:50,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 2d9b639d709a2e4f040a3e7eadeae633: 2024-12-05T07:41:50,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:50,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-05T07:41:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-05T07:41:50,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-05T07:41:50,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3090 sec 2024-12-05T07:41:50,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3120 sec 2024-12-05T07:41:52,574 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T07:41:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T07:41:52,597 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1996 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5983 rows 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2002 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6006 rows 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1986 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5952 rows 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2000 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5995 rows 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1999 2024-12-05T07:41:52,597 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5994 rows 2024-12-05T07:41:52,597 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T07:41:52,597 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T07:41:52,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:52,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:52,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:41:52,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T07:41:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T07:41:52,604 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384512604"}]},"ts":"1733384512604"} 2024-12-05T07:41:52,606 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T07:41:52,606 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T07:41:52,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T07:41:52,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, UNASSIGN}] 2024-12-05T07:41:52,610 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, UNASSIGN 2024-12-05T07:41:52,611 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=2d9b639d709a2e4f040a3e7eadeae633, regionState=CLOSING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:52,612 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, UNASSIGN because future has completed 2024-12-05T07:41:52,612 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T07:41:52,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:41:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T07:41:52,766 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(122): Close 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:52,766 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T07:41:52,766 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1722): Closing 2d9b639d709a2e4f040a3e7eadeae633, disabling compactions & flushes 2024-12-05T07:41:52,766 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:52,766 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:52,766 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. after waiting 0 ms 2024-12-05T07:41:52,766 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:52,767 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(2902): Flushing 2d9b639d709a2e4f040a3e7eadeae633 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-05T07:41:52,779 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205146d4b498d214947848f2df4e6d125f5_2d9b639d709a2e4f040a3e7eadeae633 is 50, key is test_row_0/A:col10/1733384509317/Put/seqid=0 2024-12-05T07:41:52,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742221_1397 (size=12454) 2024-12-05T07:41:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T07:41:53,186 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:53,195 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205146d4b498d214947848f2df4e6d125f5_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205146d4b498d214947848f2df4e6d125f5_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:53,196 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/48c0c2dc9c18410fbcd23555c4eb6f12, store: [table=TestAcidGuarantees family=A region=2d9b639d709a2e4f040a3e7eadeae633] 2024-12-05T07:41:53,197 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/48c0c2dc9c18410fbcd23555c4eb6f12 is 175, key is test_row_0/A:col10/1733384509317/Put/seqid=0 2024-12-05T07:41:53,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742222_1398 (size=31255) 2024-12-05T07:41:53,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T07:41:53,604 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/48c0c2dc9c18410fbcd23555c4eb6f12 2024-12-05T07:41:53,616 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/29f32295acc14576b8b781a08515ed58 is 50, key is test_row_0/B:col10/1733384509317/Put/seqid=0 2024-12-05T07:41:53,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742223_1399 (size=12301) 2024-12-05T07:41:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T07:41:54,021 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/29f32295acc14576b8b781a08515ed58 2024-12-05T07:41:54,034 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/b6e14e91ca6f443ea20e6e01a0f2dc04 is 50, key is test_row_0/C:col10/1733384509317/Put/seqid=0 2024-12-05T07:41:54,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742224_1400 (size=12301) 2024-12-05T07:41:54,439 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/b6e14e91ca6f443ea20e6e01a0f2dc04 2024-12-05T07:41:54,449 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/A/48c0c2dc9c18410fbcd23555c4eb6f12 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/48c0c2dc9c18410fbcd23555c4eb6f12 2024-12-05T07:41:54,454 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/48c0c2dc9c18410fbcd23555c4eb6f12, entries=150, sequenceid=322, filesize=30.5 K 2024-12-05T07:41:54,455 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/B/29f32295acc14576b8b781a08515ed58 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/29f32295acc14576b8b781a08515ed58 2024-12-05T07:41:54,460 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/29f32295acc14576b8b781a08515ed58, entries=150, sequenceid=322, filesize=12.0 K 2024-12-05T07:41:54,461 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/.tmp/C/b6e14e91ca6f443ea20e6e01a0f2dc04 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/b6e14e91ca6f443ea20e6e01a0f2dc04 2024-12-05T07:41:54,466 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/b6e14e91ca6f443ea20e6e01a0f2dc04, entries=150, sequenceid=322, filesize=12.0 K 2024-12-05T07:41:54,467 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2d9b639d709a2e4f040a3e7eadeae633 in 1701ms, sequenceid=322, compaction requested=true 2024-12-05T07:41:54,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/69e9f21441c14e6896c0af27f6e61a70, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/43edd6263e534f98bd50b1042d14f794, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/c0c5d14d36344acdb997d18467df9b57, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8f57610c73cb43f7aaae0d50453dae42, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/7b7d060e22f645dcb8e93d38b94d396b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f7538354a54d4c99acadb1c85bed146d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/73e17bf52f6d4cd7bb945976ea4ead82, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1faefe7390594b32ae72d6b54546e710, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/4ae9371e74a04414ac10340a5199bda9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/b84e6227ee5347488cdb816e185b6013, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/410bf201bad3468d93afac5419986247, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3d5f9990df84633aa39be2f8e568ea3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/cef59acae12e4374945e6af5778e1084, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1b8cfca0e1f842c8a1a74d50a546df9e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/82fb9bd1bc46412b968047793ddc5d08, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f0d38699b5d849f086b322db62f74d61, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/ac8fc155a4444cd0a9f09a814e30af03, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8e08c62142a74012b90b9a6808af5b6e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3738cce963d40218703f903d1a967ca, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/d72506a3fa65497192b6a1a411475147] to archive 2024-12-05T07:41:54,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:41:54,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/69e9f21441c14e6896c0af27f6e61a70 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/69e9f21441c14e6896c0af27f6e61a70 2024-12-05T07:41:54,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/43edd6263e534f98bd50b1042d14f794 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/43edd6263e534f98bd50b1042d14f794 2024-12-05T07:41:54,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/c0c5d14d36344acdb997d18467df9b57 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/c0c5d14d36344acdb997d18467df9b57 2024-12-05T07:41:54,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8f57610c73cb43f7aaae0d50453dae42 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8f57610c73cb43f7aaae0d50453dae42 2024-12-05T07:41:54,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/7b7d060e22f645dcb8e93d38b94d396b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/7b7d060e22f645dcb8e93d38b94d396b 2024-12-05T07:41:54,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f7538354a54d4c99acadb1c85bed146d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f7538354a54d4c99acadb1c85bed146d 2024-12-05T07:41:54,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/73e17bf52f6d4cd7bb945976ea4ead82 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/73e17bf52f6d4cd7bb945976ea4ead82 2024-12-05T07:41:54,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1faefe7390594b32ae72d6b54546e710 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1faefe7390594b32ae72d6b54546e710 2024-12-05T07:41:54,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/4ae9371e74a04414ac10340a5199bda9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/4ae9371e74a04414ac10340a5199bda9 2024-12-05T07:41:54,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/b84e6227ee5347488cdb816e185b6013 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/b84e6227ee5347488cdb816e185b6013 2024-12-05T07:41:54,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/410bf201bad3468d93afac5419986247 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/410bf201bad3468d93afac5419986247 2024-12-05T07:41:54,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3d5f9990df84633aa39be2f8e568ea3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3d5f9990df84633aa39be2f8e568ea3 2024-12-05T07:41:54,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/cef59acae12e4374945e6af5778e1084 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/cef59acae12e4374945e6af5778e1084 2024-12-05T07:41:54,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1b8cfca0e1f842c8a1a74d50a546df9e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/1b8cfca0e1f842c8a1a74d50a546df9e 2024-12-05T07:41:54,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/82fb9bd1bc46412b968047793ddc5d08 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/82fb9bd1bc46412b968047793ddc5d08 2024-12-05T07:41:54,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f0d38699b5d849f086b322db62f74d61 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f0d38699b5d849f086b322db62f74d61 2024-12-05T07:41:54,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/ac8fc155a4444cd0a9f09a814e30af03 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/ac8fc155a4444cd0a9f09a814e30af03 2024-12-05T07:41:54,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8e08c62142a74012b90b9a6808af5b6e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/8e08c62142a74012b90b9a6808af5b6e 2024-12-05T07:41:54,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3738cce963d40218703f903d1a967ca to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/f3738cce963d40218703f903d1a967ca 2024-12-05T07:41:54,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/d72506a3fa65497192b6a1a411475147 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/d72506a3fa65497192b6a1a411475147 2024-12-05T07:41:54,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/78bfe145415446b09e868f262f62bafd, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f5c7a207c86a479693964231d577ddf1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f83590927ee349ba92fff1429ad0f5f2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/fc874f2b32fa4a4fb294369ce2e93dbc, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/9fc91ecee8824c2695d228c11bd2fda1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/c5129ce2ba1f44a58f6a5132e99e5c31, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/16924b960c4d486fbdf79f7638cea0de, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/73a6c57947654555977a1698c047cfd1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f54b2b6c0b1f4651800b8802898d9b35, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f1e3f8e5e3f44c24bbdb1de5e523222b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/5b7477658598455bb81f534c40f5af1e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/d0d129fa1937453ba8e762befc38e3f7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/50dcadefdb81466b99865181f7a46b84, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/35fe0fd4b5c54494b18115686b6d7331, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/7a0f3d89d26e4d1098dc95020bd97f8e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/35740b69b07441a8a0766d98c08c0ea7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/58e597507cfa4d5fad13e593e2cb4d0c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/8417c03ec5c34c55a2140b6a43078c41, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/6fd003a05d3a4aadb4b12cdebb93352e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/e8a8f20a540f4b6aa9c3ade00c882d6b] to archive 2024-12-05T07:41:54,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:41:54,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/78bfe145415446b09e868f262f62bafd to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/78bfe145415446b09e868f262f62bafd 2024-12-05T07:41:54,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f5c7a207c86a479693964231d577ddf1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f5c7a207c86a479693964231d577ddf1 2024-12-05T07:41:54,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f83590927ee349ba92fff1429ad0f5f2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f83590927ee349ba92fff1429ad0f5f2 2024-12-05T07:41:54,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/fc874f2b32fa4a4fb294369ce2e93dbc to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/fc874f2b32fa4a4fb294369ce2e93dbc 2024-12-05T07:41:54,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/9fc91ecee8824c2695d228c11bd2fda1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/9fc91ecee8824c2695d228c11bd2fda1 2024-12-05T07:41:54,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/c5129ce2ba1f44a58f6a5132e99e5c31 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/c5129ce2ba1f44a58f6a5132e99e5c31 2024-12-05T07:41:54,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/16924b960c4d486fbdf79f7638cea0de to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/16924b960c4d486fbdf79f7638cea0de 2024-12-05T07:41:54,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/73a6c57947654555977a1698c047cfd1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/73a6c57947654555977a1698c047cfd1 2024-12-05T07:41:54,507 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f54b2b6c0b1f4651800b8802898d9b35 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f54b2b6c0b1f4651800b8802898d9b35 2024-12-05T07:41:54,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f1e3f8e5e3f44c24bbdb1de5e523222b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/f1e3f8e5e3f44c24bbdb1de5e523222b 2024-12-05T07:41:54,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/5b7477658598455bb81f534c40f5af1e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/5b7477658598455bb81f534c40f5af1e 2024-12-05T07:41:54,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/d0d129fa1937453ba8e762befc38e3f7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/d0d129fa1937453ba8e762befc38e3f7 2024-12-05T07:41:54,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/50dcadefdb81466b99865181f7a46b84 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/50dcadefdb81466b99865181f7a46b84 2024-12-05T07:41:54,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/35fe0fd4b5c54494b18115686b6d7331 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/35fe0fd4b5c54494b18115686b6d7331 2024-12-05T07:41:54,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/7a0f3d89d26e4d1098dc95020bd97f8e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/7a0f3d89d26e4d1098dc95020bd97f8e 2024-12-05T07:41:54,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/35740b69b07441a8a0766d98c08c0ea7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/35740b69b07441a8a0766d98c08c0ea7 2024-12-05T07:41:54,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/58e597507cfa4d5fad13e593e2cb4d0c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/58e597507cfa4d5fad13e593e2cb4d0c 2024-12-05T07:41:54,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/8417c03ec5c34c55a2140b6a43078c41 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/8417c03ec5c34c55a2140b6a43078c41 2024-12-05T07:41:54,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/6fd003a05d3a4aadb4b12cdebb93352e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/6fd003a05d3a4aadb4b12cdebb93352e 2024-12-05T07:41:54,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/e8a8f20a540f4b6aa9c3ade00c882d6b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/e8a8f20a540f4b6aa9c3ade00c882d6b 2024-12-05T07:41:54,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/98b80eb7839343319f9956c2390aaae1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/ae7e57c29a7641298f309a9ae762560d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/43eb62f7a0074a079b17730bff9ddc9d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/c5837530abfd4b4eb9bc3a9cca5a5f44, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/87b74ce7afe54c688ff07f7a58d3de75, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/d55ae993a98d440d988eaa91bac06633, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f34e3e6158cb4157ae59461c53ab7918, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/356f1065a33b4ded8f5692dea7ea2f17, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/fc7ef22525494f4c82d101561eb183a7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3d9a9dba803143ccb36c682ed8c553a5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/6944bc3e4d20405187bbf9879925d5df, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/eed992edb82a4ef39f04cabf10c4c030, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/7b7704b5ea0d41bbb7fdf3b47e23b569, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/2be155c48b534714ab45325b470a85d3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f930584cb7c64e288eb125c9db788ef9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/785dfedb64d44372823e3a3bfbbdd24a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f902204b278d41c6b767fde0a0aeb3cf, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3b6f5aaf543f42dbab104451c639f659, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/79bb8b6911c9400a83505b8135e285e3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/4e0fd46ce32343c4bddf2a98d74b808b] to archive 2024-12-05T07:41:54,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:41:54,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/98b80eb7839343319f9956c2390aaae1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/98b80eb7839343319f9956c2390aaae1 2024-12-05T07:41:54,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/ae7e57c29a7641298f309a9ae762560d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/ae7e57c29a7641298f309a9ae762560d 2024-12-05T07:41:54,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/43eb62f7a0074a079b17730bff9ddc9d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/43eb62f7a0074a079b17730bff9ddc9d 2024-12-05T07:41:54,528 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/c5837530abfd4b4eb9bc3a9cca5a5f44 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/c5837530abfd4b4eb9bc3a9cca5a5f44 2024-12-05T07:41:54,530 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/87b74ce7afe54c688ff07f7a58d3de75 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/87b74ce7afe54c688ff07f7a58d3de75 2024-12-05T07:41:54,531 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/d55ae993a98d440d988eaa91bac06633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/d55ae993a98d440d988eaa91bac06633 2024-12-05T07:41:54,532 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f34e3e6158cb4157ae59461c53ab7918 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f34e3e6158cb4157ae59461c53ab7918 2024-12-05T07:41:54,533 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/356f1065a33b4ded8f5692dea7ea2f17 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/356f1065a33b4ded8f5692dea7ea2f17 2024-12-05T07:41:54,534 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/fc7ef22525494f4c82d101561eb183a7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/fc7ef22525494f4c82d101561eb183a7 2024-12-05T07:41:54,535 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3d9a9dba803143ccb36c682ed8c553a5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3d9a9dba803143ccb36c682ed8c553a5 2024-12-05T07:41:54,536 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/6944bc3e4d20405187bbf9879925d5df to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/6944bc3e4d20405187bbf9879925d5df 2024-12-05T07:41:54,537 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/eed992edb82a4ef39f04cabf10c4c030 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/eed992edb82a4ef39f04cabf10c4c030 2024-12-05T07:41:54,539 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/7b7704b5ea0d41bbb7fdf3b47e23b569 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/7b7704b5ea0d41bbb7fdf3b47e23b569 2024-12-05T07:41:54,540 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/2be155c48b534714ab45325b470a85d3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/2be155c48b534714ab45325b470a85d3 2024-12-05T07:41:54,541 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f930584cb7c64e288eb125c9db788ef9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f930584cb7c64e288eb125c9db788ef9 2024-12-05T07:41:54,542 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/785dfedb64d44372823e3a3bfbbdd24a to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/785dfedb64d44372823e3a3bfbbdd24a 2024-12-05T07:41:54,543 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f902204b278d41c6b767fde0a0aeb3cf to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/f902204b278d41c6b767fde0a0aeb3cf 2024-12-05T07:41:54,544 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3b6f5aaf543f42dbab104451c639f659 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/3b6f5aaf543f42dbab104451c639f659 2024-12-05T07:41:54,545 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/79bb8b6911c9400a83505b8135e285e3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/79bb8b6911c9400a83505b8135e285e3 2024-12-05T07:41:54,546 DEBUG [StoreCloser-TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/4e0fd46ce32343c4bddf2a98d74b808b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/4e0fd46ce32343c4bddf2a98d74b808b 2024-12-05T07:41:54,551 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/recovered.edits/325.seqid, newMaxSeqId=325, maxSeqId=4 2024-12-05T07:41:54,551 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. 2024-12-05T07:41:54,552 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] regionserver.HRegion(1676): Region close journal for 2d9b639d709a2e4f040a3e7eadeae633: Waiting for close lock at 1733384512766Running coprocessor pre-close hooks at 1733384512766Disabling compacts and flushes for region at 1733384512766Disabling writes for close at 1733384512766Obtaining lock to block concurrent updates at 1733384512767 (+1 ms)Preparing flush snapshotting stores in 2d9b639d709a2e4f040a3e7eadeae633 at 1733384512767Finished memstore snapshotting TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733384512767Flushing stores of TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633. at 1733384512769 (+2 ms)Flushing 2d9b639d709a2e4f040a3e7eadeae633/A: creating writer at 1733384512769Flushing mob file 2d9b639d709a2e4f040a3e7eadeae633/A: appending metadata at 1733384512779 (+10 ms)Flushing mob file 2d9b639d709a2e4f040a3e7eadeae633/A: closing flushed file at 1733384512779Flushing 2d9b639d709a2e4f040a3e7eadeae633/A: appending metadata at 1733384513196 (+417 ms)Flushing 2d9b639d709a2e4f040a3e7eadeae633/A: closing flushed file at 1733384513197 (+1 ms)Flushing 2d9b639d709a2e4f040a3e7eadeae633/B: creating writer at 1733384513610 (+413 ms)Flushing 2d9b639d709a2e4f040a3e7eadeae633/B: appending metadata at 1733384513615 (+5 ms)Flushing 2d9b639d709a2e4f040a3e7eadeae633/B: closing flushed file at 1733384513615Flushing 2d9b639d709a2e4f040a3e7eadeae633/C: creating writer at 1733384514029 (+414 ms)Flushing 2d9b639d709a2e4f040a3e7eadeae633/C: appending metadata at 1733384514033 (+4 ms)Flushing 2d9b639d709a2e4f040a3e7eadeae633/C: closing flushed file at 1733384514033Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@735292f5: reopening flushed file at 1733384514448 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@380e5f14: reopening flushed file at 1733384514454 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@274d664a: reopening flushed file at 1733384514460 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2d9b639d709a2e4f040a3e7eadeae633 in 1701ms, sequenceid=322, compaction requested=true at 1733384514467 (+7 ms)Writing region close event to WAL at 1733384514548 (+81 ms)Running coprocessor post-close hooks at 1733384514551 (+3 ms)Closed at 1733384514551 2024-12-05T07:41:54,554 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=106}] handler.UnassignRegionHandler(157): Closed 2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,555 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=2d9b639d709a2e4f040a3e7eadeae633, regionState=CLOSED 2024-12-05T07:41:54,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:41:54,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-05T07:41:54,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseRegionProcedure 2d9b639d709a2e4f040a3e7eadeae633, server=fc6cd38557f3,36693,1733384394401 in 1.9450 sec 2024-12-05T07:41:54,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-12-05T07:41:54,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d9b639d709a2e4f040a3e7eadeae633, UNASSIGN in 1.9500 sec 2024-12-05T07:41:54,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-05T07:41:54,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9550 sec 2024-12-05T07:41:54,564 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384514564"}]},"ts":"1733384514564"} 2024-12-05T07:41:54,565 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T07:41:54,565 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T07:41:54,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9670 sec 2024-12-05T07:41:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T07:41:54,757 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:54,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T07:41:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:54,763 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=107, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T07:41:54,764 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=107, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:54,767 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,771 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/recovered.edits] 2024-12-05T07:41:54,774 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/48c0c2dc9c18410fbcd23555c4eb6f12 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/48c0c2dc9c18410fbcd23555c4eb6f12 2024-12-05T07:41:54,776 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/654ea0f2fcd14d62b0cd4e31618246cb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/654ea0f2fcd14d62b0cd4e31618246cb 2024-12-05T07:41:54,778 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/e4bda6f9fa504004b0b4b496d2676b09 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/A/e4bda6f9fa504004b0b4b496d2676b09 2024-12-05T07:41:54,781 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/29f32295acc14576b8b781a08515ed58 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/29f32295acc14576b8b781a08515ed58 2024-12-05T07:41:54,783 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/c072f9a7966b4c3397229a59b0540c60 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/c072f9a7966b4c3397229a59b0540c60 2024-12-05T07:41:54,784 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/fcac38ebf1a0447db6ba5a2066a0d36f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/B/fcac38ebf1a0447db6ba5a2066a0d36f 2024-12-05T07:41:54,788 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/b6e14e91ca6f443ea20e6e01a0f2dc04 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/b6e14e91ca6f443ea20e6e01a0f2dc04 2024-12-05T07:41:54,790 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/bb37a08232b546f289650a0be7de0699 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/bb37a08232b546f289650a0be7de0699 2024-12-05T07:41:54,792 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/ed413bf346304ac9a7f4e210d732e855 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/C/ed413bf346304ac9a7f4e210d732e855 2024-12-05T07:41:54,796 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/recovered.edits/325.seqid to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633/recovered.edits/325.seqid 2024-12-05T07:41:54,797 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,797 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T07:41:54,798 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T07:41:54,799 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T07:41:54,804 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120508a5e8959df1401a9bc9a2cef20a9733_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120508a5e8959df1401a9bc9a2cef20a9733_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,806 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205146d4b498d214947848f2df4e6d125f5_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205146d4b498d214947848f2df4e6d125f5_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,808 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120518c9ba38325c400cbafaae753815c026_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120518c9ba38325c400cbafaae753815c026_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,810 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051b9bd6da7a704a42b7833432d9c07d06_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051b9bd6da7a704a42b7833432d9c07d06_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,812 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051bc8fd0b8e824a89955fc657dbe767a5_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051bc8fd0b8e824a89955fc657dbe767a5_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,814 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120522ee2d66a30e4335be87698085d77344_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120522ee2d66a30e4335be87698085d77344_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,862 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120529b16b44a3c3429aa4e6d49014b7119f_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120529b16b44a3c3429aa4e6d49014b7119f_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,865 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205353f0476124e4464af419ae12a1ca524_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205353f0476124e4464af419ae12a1ca524_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,869 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053a7bc52ab26b446a870819fd96a8f592_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053a7bc52ab26b446a870819fd96a8f592_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,872 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054d1c3fb8ef60431b90b510ecbcaca11e_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054d1c3fb8ef60431b90b510ecbcaca11e_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,874 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056a69602608e04824add89f964a4bd6c7_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056a69602608e04824add89f964a4bd6c7_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,876 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056f8227b933634426b202a0564d97420e_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412056f8227b933634426b202a0564d97420e_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T07:41:54,878 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057a44a62d455f47289ae330b739408e9a_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057a44a62d455f47289ae330b739408e9a_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,880 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059b191ed0bf42487fa4f5825ee13b125b_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059b191ed0bf42487fa4f5825ee13b125b_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,881 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c41f8b850e4846ba985a5c19af8177eb_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c41f8b850e4846ba985a5c19af8177eb_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,883 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e3b1836cb35747a58422e46953f58b8e_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e3b1836cb35747a58422e46953f58b8e_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,884 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f969a21e5aa24008bf125e4b127c5438_2d9b639d709a2e4f040a3e7eadeae633 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f969a21e5aa24008bf125e4b127c5438_2d9b639d709a2e4f040a3e7eadeae633 2024-12-05T07:41:54,885 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T07:41:54,887 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=107, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:54,889 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T07:41:54,892 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T07:41:54,893 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=107, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:54,893 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T07:41:54,893 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733384514893"}]},"ts":"9223372036854775807"} 2024-12-05T07:41:54,896 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T07:41:54,896 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2d9b639d709a2e4f040a3e7eadeae633, NAME => 'TestAcidGuarantees,,1733384485280.2d9b639d709a2e4f040a3e7eadeae633.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T07:41:54,896 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T07:41:54,896 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733384514896"}]},"ts":"9223372036854775807"} 2024-12-05T07:41:54,898 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T07:41:54,898 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=107, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:54,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 139 msec 2024-12-05T07:41:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-05T07:41:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T07:41:55,088 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:55,103 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 238) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=410 (was 501), ProcessCount=11 (was 11), AvailableMemoryMB=6726 (was 6790) 2024-12-05T07:41:55,111 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=410, ProcessCount=11, AvailableMemoryMB=6726 2024-12-05T07:41:55,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T07:41:55,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T07:41:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T07:41:55,114 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T07:41:55,114 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:55,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 108 2024-12-05T07:41:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T07:41:55,115 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T07:41:55,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742225_1401 (size=959) 2024-12-05T07:41:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T07:41:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T07:41:55,524 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15 2024-12-05T07:41:55,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742226_1402 (size=53) 2024-12-05T07:41:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T07:41:55,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:41:55,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 45734b659127c96dcd616039a555b19f, disabling compactions & flushes 2024-12-05T07:41:55,935 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:55,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:55,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. after waiting 0 ms 2024-12-05T07:41:55,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:55,935 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:55,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 45734b659127c96dcd616039a555b19f: Waiting for close lock at 1733384515935Disabling compacts and flushes for region at 1733384515935Disabling writes for close at 1733384515935Writing region close event to WAL at 1733384515935Closed at 1733384515935 2024-12-05T07:41:55,938 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T07:41:55,939 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733384515938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733384515938"}]},"ts":"1733384515938"} 2024-12-05T07:41:55,943 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T07:41:55,944 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T07:41:55,944 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384515944"}]},"ts":"1733384515944"} 2024-12-05T07:41:55,946 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T07:41:55,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45734b659127c96dcd616039a555b19f, ASSIGN}] 2024-12-05T07:41:55,948 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45734b659127c96dcd616039a555b19f, ASSIGN 2024-12-05T07:41:55,949 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45734b659127c96dcd616039a555b19f, ASSIGN; state=OFFLINE, location=fc6cd38557f3,36693,1733384394401; forceNewPlan=false, retain=false 2024-12-05T07:41:56,100 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=45734b659127c96dcd616039a555b19f, regionState=OPENING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:56,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45734b659127c96dcd616039a555b19f, ASSIGN because future has completed 2024-12-05T07:41:56,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:41:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T07:41:56,264 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:56,264 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7752): Opening region: {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} 2024-12-05T07:41:56,266 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,266 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:41:56,266 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7794): checking encryption for 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,266 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(7797): checking classloading for 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,268 INFO [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,269 INFO [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45734b659127c96dcd616039a555b19f columnFamilyName A 2024-12-05T07:41:56,269 DEBUG [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:56,270 INFO [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] regionserver.HStore(327): Store=45734b659127c96dcd616039a555b19f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:41:56,270 INFO [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,271 INFO [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45734b659127c96dcd616039a555b19f columnFamilyName B 2024-12-05T07:41:56,271 DEBUG [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:56,271 INFO [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] regionserver.HStore(327): Store=45734b659127c96dcd616039a555b19f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:41:56,271 INFO [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,272 INFO [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45734b659127c96dcd616039a555b19f columnFamilyName C 2024-12-05T07:41:56,272 DEBUG [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:41:56,273 INFO [StoreOpener-45734b659127c96dcd616039a555b19f-1 {}] regionserver.HStore(327): Store=45734b659127c96dcd616039a555b19f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:41:56,273 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1038): replaying wal for 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,274 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,274 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,275 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1048): stopping wal replay for 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,275 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1060): Cleaning up temporary data for 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,275 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T07:41:56,277 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1093): writing seq id for 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,278 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T07:41:56,279 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1114): Opened 45734b659127c96dcd616039a555b19f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69371844, jitterRate=0.03372102975845337}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T07:41:56,279 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:56,279 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegion(1006): Region open journal for 45734b659127c96dcd616039a555b19f: Running coprocessor pre-open hook at 1733384516266Writing region info on filesystem at 1733384516266Initializing all the Stores at 1733384516267 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384516267Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384516267Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384516267Cleaning up temporary data from old regions at 1733384516275 (+8 ms)Running coprocessor post-open hooks at 1733384516279 (+4 ms)Region opened successfully at 1733384516279 2024-12-05T07:41:56,280 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., pid=110, masterSystemTime=1733384516256 2024-12-05T07:41:56,282 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:56,282 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=110}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:56,283 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=45734b659127c96dcd616039a555b19f, regionState=OPEN, openSeqNum=2, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:56,285 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:41:56,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-05T07:41:56,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; OpenRegionProcedure 45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 in 182 msec 2024-12-05T07:41:56,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-12-05T07:41:56,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45734b659127c96dcd616039a555b19f, ASSIGN in 341 msec 2024-12-05T07:41:56,291 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T07:41:56,291 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384516291"}]},"ts":"1733384516291"} 2024-12-05T07:41:56,292 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T07:41:56,292 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=108, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T07:41:56,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-12-05T07:41:56,609 WARN [regionserver/fc6cd38557f3:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-05T07:41:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-05T07:41:57,266 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:41:57,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:57,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:57,270 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce39c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:57,273 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:57,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba06479, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,274 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:57,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:57,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:57,277 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T07:41:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is fc6cd38557f3,33643,1733384393481 2024-12-05T07:41:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@22bdfbaa 2024-12-05T07:41:57,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T07:41:57,279 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T07:41:57,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:57,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:57,282 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d37cb67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,282 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:57,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:57,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:57,283 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:57,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a9d2d28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:57,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:57,286 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:57,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc1d0ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:57,288 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:57,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586a57d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,290 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:57,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:57,291 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d57b589, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:57,292 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26900abb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:57,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:57,293 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:57,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@134ce117, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:57,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:57,294 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a045f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:57,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:57,296 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64267a7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:57,297 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@290dd49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:57,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:57,299 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f07cbfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:57,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:57,300 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:57,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5371fa54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,301 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:57,301 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:57,301 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:57,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e502979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:57,302 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:57,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b4fa6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:57,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:57,304 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:57,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:57,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:57,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64eca3a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:57,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:57,304 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:57,305 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:57,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@760b3123, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:57,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:57,306 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713da305, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:57,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:57,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f278d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109f3221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:41:57,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:41:57,308 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:41:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:41:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:41:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213fe42c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:41:57,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:41:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:41:57,309 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:41:57,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eac2cf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:41:57,311 DEBUG [Thread-1824 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:57,311 DEBUG [Thread-1835 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:57,311 DEBUG [Thread-1828 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:57,311 DEBUG [Thread-1830 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:57,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:57,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:57,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:57,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:41:57,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:41:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:41:57,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:41:57,315 DEBUG [Thread-1841 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:57,316 DEBUG [Thread-1839 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:57,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:57,316 DEBUG [Thread-1839 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:57,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:57,317 DEBUG [Thread-1832 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:57,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:57,318 DEBUG [Thread-1837 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:41:57,318 DEBUG [Thread-1826 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:57,318 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:57,319 DEBUG [Thread-1843 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:41:57,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:57,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:57,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:57,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:57,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:57,320 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:41:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:41:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:41:57,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:41:57,320 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:57,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:57,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:57,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:41:57,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:41:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:41:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:41:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-05T07:41:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2] 2024-12-05T07:41:57,324 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:41:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:57,324 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:57,324 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:41:57,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T07:41:57,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:41:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T07:41:57,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/a98f5922e0384cac9045b69b7d1a6bfb is 50, key is test_row_0/A:col10/1733384517317/Put/seqid=0 2024-12-05T07:41:57,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742227_1403 (size=16681) 2024-12-05T07:41:57,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384577350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384577350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384577351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384577355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384577355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T07:41:57,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384577466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384577466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384577466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384577466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384577466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-05T07:41:57,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:57,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:41:57,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:57,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-05T07:41:57,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:57,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:41:57,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:57,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T07:41:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384577676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384577676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384577676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384577676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384577676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/a98f5922e0384cac9045b69b7d1a6bfb 2024-12-05T07:41:57,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/bc68405e53df4d489b2d3450a7c513d1 is 50, key is test_row_0/B:col10/1733384517317/Put/seqid=0 2024-12-05T07:41:57,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742228_1404 (size=12001) 2024-12-05T07:41:57,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-05T07:41:57,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:57,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:41:57,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:57,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-05T07:41:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:41:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:57,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:57,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T07:41:57,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384577986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384577986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384577986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384577986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384577986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:57,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-05T07:41:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:41:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:58,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:58,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:58,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:41:58,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/bc68405e53df4d489b2d3450a7c513d1 2024-12-05T07:41:58,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/57cf46f983e243c8b4b05671061ccae7 is 50, key is test_row_0/C:col10/1733384517317/Put/seqid=0 2024-12-05T07:41:58,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742229_1405 (size=12001) 2024-12-05T07:41:58,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/57cf46f983e243c8b4b05671061ccae7 2024-12-05T07:41:58,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/a98f5922e0384cac9045b69b7d1a6bfb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a98f5922e0384cac9045b69b7d1a6bfb 2024-12-05T07:41:58,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a98f5922e0384cac9045b69b7d1a6bfb, entries=250, sequenceid=13, filesize=16.3 K 2024-12-05T07:41:58,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/bc68405e53df4d489b2d3450a7c513d1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/bc68405e53df4d489b2d3450a7c513d1 2024-12-05T07:41:58,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/bc68405e53df4d489b2d3450a7c513d1, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T07:41:58,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/57cf46f983e243c8b4b05671061ccae7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/57cf46f983e243c8b4b05671061ccae7 2024-12-05T07:41:58,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/57cf46f983e243c8b4b05671061ccae7, entries=150, sequenceid=13, filesize=11.7 K 2024-12-05T07:41:58,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 45734b659127c96dcd616039a555b19f in 890ms, sequenceid=13, compaction requested=false 2024-12-05T07:41:58,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:41:58,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-05T07:41:58,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:58,242 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T07:41:58,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/10439ee63a9948d2abcc0a95fc1d6364 is 50, key is test_row_0/A:col10/1733384517354/Put/seqid=0 2024-12-05T07:41:58,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742230_1406 (size=12001) 2024-12-05T07:41:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T07:41:58,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:41:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:58,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384578506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384578509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384578510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384578511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384578512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384578616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384578626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384578626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384578626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,651 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/10439ee63a9948d2abcc0a95fc1d6364 2024-12-05T07:41:58,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/7adce838cd984f388ec1816e8e1bb757 is 50, key is test_row_0/B:col10/1733384517354/Put/seqid=0 2024-12-05T07:41:58,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742231_1407 (size=12001) 2024-12-05T07:41:58,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384578826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384578836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384578836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384578836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:58,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:58,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:59,061 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/7adce838cd984f388ec1816e8e1bb757 2024-12-05T07:41:59,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/dc5b707e97d348519a9bbd8163059284 is 50, key is test_row_0/C:col10/1733384517354/Put/seqid=0 2024-12-05T07:41:59,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742232_1408 (size=12001) 2024-12-05T07:41:59,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384579136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:59,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384579146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:59,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384579146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:59,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384579146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:59,470 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/dc5b707e97d348519a9bbd8163059284 2024-12-05T07:41:59,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/10439ee63a9948d2abcc0a95fc1d6364 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/10439ee63a9948d2abcc0a95fc1d6364 2024-12-05T07:41:59,475 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/10439ee63a9948d2abcc0a95fc1d6364, entries=150, sequenceid=37, filesize=11.7 K 2024-12-05T07:41:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T07:41:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/7adce838cd984f388ec1816e8e1bb757 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7adce838cd984f388ec1816e8e1bb757 2024-12-05T07:41:59,480 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7adce838cd984f388ec1816e8e1bb757, entries=150, sequenceid=37, filesize=11.7 K 2024-12-05T07:41:59,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/dc5b707e97d348519a9bbd8163059284 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/dc5b707e97d348519a9bbd8163059284 2024-12-05T07:41:59,484 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/dc5b707e97d348519a9bbd8163059284, entries=150, sequenceid=37, filesize=11.7 K 2024-12-05T07:41:59,485 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 45734b659127c96dcd616039a555b19f in 1242ms, sequenceid=37, compaction requested=false 2024-12-05T07:41:59,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:41:59,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:41:59,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-05T07:41:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-05T07:41:59,487 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-05T07:41:59,487 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1610 sec 2024-12-05T07:41:59,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.1680 sec 2024-12-05T07:41:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:41:59,529 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:41:59,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T07:41:59,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/18fbd2cf50fb4090bfaaf0dc0a6356a5 is 50, key is test_row_0/A:col10/1733384519528/Put/seqid=0 2024-12-05T07:41:59,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742233_1409 (size=14341) 2024-12-05T07:41:59,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384579645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:59,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:59,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:59,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384579650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:59,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:59,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384579655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:59,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:59,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384579656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:59,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:59,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384579666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:59,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:59,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384579766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:41:59,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/18fbd2cf50fb4090bfaaf0dc0a6356a5 2024-12-05T07:41:59,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/7ef69cff6dc24085a7b83657673af1d8 is 50, key is test_row_0/B:col10/1733384519528/Put/seqid=0 2024-12-05T07:41:59,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742234_1410 (size=12001) 2024-12-05T07:41:59,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:41:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384579976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:41:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:41:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:00,067 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T07:42:00,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384580286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:00,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/7ef69cff6dc24085a7b83657673af1d8 2024-12-05T07:42:00,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/215bd4e650b74701928d556689b1a04e is 50, key is test_row_0/C:col10/1733384519528/Put/seqid=0 2024-12-05T07:42:00,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742235_1411 (size=12001) 2024-12-05T07:42:00,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384580655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:00,658 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 45734b659127c96dcd616039a555b19f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:00,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384580665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:00,670 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 45734b659127c96dcd616039a555b19f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:00,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384580676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:00,678 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in 45734b659127c96dcd616039a555b19f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:00,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384580686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:00,691 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 45734b659127c96dcd616039a555b19f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:00,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:00,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/215bd4e650b74701928d556689b1a04e 2024-12-05T07:42:00,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/18fbd2cf50fb4090bfaaf0dc0a6356a5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/18fbd2cf50fb4090bfaaf0dc0a6356a5 2024-12-05T07:42:00,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/18fbd2cf50fb4090bfaaf0dc0a6356a5, entries=200, sequenceid=50, filesize=14.0 K 2024-12-05T07:42:00,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/7ef69cff6dc24085a7b83657673af1d8 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7ef69cff6dc24085a7b83657673af1d8 2024-12-05T07:42:00,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7ef69cff6dc24085a7b83657673af1d8, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T07:42:00,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/215bd4e650b74701928d556689b1a04e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/215bd4e650b74701928d556689b1a04e 2024-12-05T07:42:00,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/215bd4e650b74701928d556689b1a04e, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T07:42:00,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 45734b659127c96dcd616039a555b19f in 1241ms, sequenceid=50, compaction requested=true 2024-12-05T07:42:00,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:00,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:00,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:00,771 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:00,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:00,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:00,771 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:00,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:00,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:00,771 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:00,771 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:00,771 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/A is initiating minor compaction (all files) 2024-12-05T07:42:00,771 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/B is initiating minor compaction (all files) 2024-12-05T07:42:00,771 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/B in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:00,771 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/A in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:00,772 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/bc68405e53df4d489b2d3450a7c513d1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7adce838cd984f388ec1816e8e1bb757, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7ef69cff6dc24085a7b83657673af1d8] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=35.2 K 2024-12-05T07:42:00,772 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a98f5922e0384cac9045b69b7d1a6bfb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/10439ee63a9948d2abcc0a95fc1d6364, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/18fbd2cf50fb4090bfaaf0dc0a6356a5] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=42.0 K 2024-12-05T07:42:00,772 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a98f5922e0384cac9045b69b7d1a6bfb, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733384517317 2024-12-05T07:42:00,772 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting bc68405e53df4d489b2d3450a7c513d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733384517317 2024-12-05T07:42:00,772 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10439ee63a9948d2abcc0a95fc1d6364, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733384517340 2024-12-05T07:42:00,772 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7adce838cd984f388ec1816e8e1bb757, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733384517340 2024-12-05T07:42:00,772 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18fbd2cf50fb4090bfaaf0dc0a6356a5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733384518504 2024-12-05T07:42:00,772 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ef69cff6dc24085a7b83657673af1d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733384518504 2024-12-05T07:42:00,779 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#B#compaction#341 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:00,780 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/859e7adbcc184490a22677dd447b308f is 50, key is test_row_0/B:col10/1733384519528/Put/seqid=0 2024-12-05T07:42:00,782 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#A#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:00,783 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/5c14fdac638746ccb547ca54125145d1 is 50, key is test_row_0/A:col10/1733384519528/Put/seqid=0 2024-12-05T07:42:00,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742236_1412 (size=12104) 2024-12-05T07:42:00,789 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/859e7adbcc184490a22677dd447b308f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/859e7adbcc184490a22677dd447b308f 2024-12-05T07:42:00,797 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/B of 45734b659127c96dcd616039a555b19f into 859e7adbcc184490a22677dd447b308f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:00,797 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:00,797 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/B, priority=13, startTime=1733384520771; duration=0sec 2024-12-05T07:42:00,797 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:00,797 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:B 2024-12-05T07:42:00,797 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:00,797 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:00,797 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/C is initiating minor compaction (all files) 2024-12-05T07:42:00,798 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/C in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:00,798 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/57cf46f983e243c8b4b05671061ccae7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/dc5b707e97d348519a9bbd8163059284, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/215bd4e650b74701928d556689b1a04e] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=35.2 K 2024-12-05T07:42:00,798 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 57cf46f983e243c8b4b05671061ccae7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733384517317 2024-12-05T07:42:00,798 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting dc5b707e97d348519a9bbd8163059284, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733384517340 2024-12-05T07:42:00,798 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 215bd4e650b74701928d556689b1a04e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733384518504 2024-12-05T07:42:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:00,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:00,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T07:42:00,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742237_1413 (size=12104) 2024-12-05T07:42:00,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/687e677a709a4c76a8da017701b2012c is 50, key is test_row_0/A:col10/1733384519649/Put/seqid=0 2024-12-05T07:42:00,805 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#C#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:00,805 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/5b09548714f0468cbc87d77f2572ee62 is 50, key is test_row_0/C:col10/1733384519528/Put/seqid=0 2024-12-05T07:42:00,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742238_1414 (size=14341) 2024-12-05T07:42:00,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/687e677a709a4c76a8da017701b2012c 2024-12-05T07:42:00,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/767ebaf8a9de4cbfb3ac203df0c21a4d is 50, key is test_row_0/B:col10/1733384519649/Put/seqid=0 2024-12-05T07:42:00,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742239_1415 (size=12104) 2024-12-05T07:42:00,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742240_1416 (size=12001) 2024-12-05T07:42:00,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384580872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:00,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:00,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384580986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:00,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:00,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:01,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384581196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:01,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:01,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:01,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:01,207 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/5c14fdac638746ccb547ca54125145d1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5c14fdac638746ccb547ca54125145d1 2024-12-05T07:42:01,210 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/A of 45734b659127c96dcd616039a555b19f into 5c14fdac638746ccb547ca54125145d1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:01,210 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:01,210 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/A, priority=13, startTime=1733384520770; duration=0sec 2024-12-05T07:42:01,211 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:01,211 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:A 2024-12-05T07:42:01,239 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/5b09548714f0468cbc87d77f2572ee62 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/5b09548714f0468cbc87d77f2572ee62 2024-12-05T07:42:01,243 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/C of 45734b659127c96dcd616039a555b19f into 5b09548714f0468cbc87d77f2572ee62(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:01,243 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:01,243 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/C, priority=13, startTime=1733384520771; duration=0sec 2024-12-05T07:42:01,243 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:01,243 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:C 2024-12-05T07:42:01,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/767ebaf8a9de4cbfb3ac203df0c21a4d 2024-12-05T07:42:01,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/0b2cef2b62ee48f28eaefc801d4cd845 is 50, key is test_row_0/C:col10/1733384519649/Put/seqid=0 2024-12-05T07:42:01,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742241_1417 (size=12001) 2024-12-05T07:42:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-05T07:42:01,496 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:01,496 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-05T07:42:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T07:42:01,498 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:01,498 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:01,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:01,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384581506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:01,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:01,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:01,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T07:42:01,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T07:42:01,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:01,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:01,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:01,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:01,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:01,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/0b2cef2b62ee48f28eaefc801d4cd845 2024-12-05T07:42:01,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/687e677a709a4c76a8da017701b2012c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/687e677a709a4c76a8da017701b2012c 2024-12-05T07:42:01,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/687e677a709a4c76a8da017701b2012c, entries=200, sequenceid=75, filesize=14.0 K 2024-12-05T07:42:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/767ebaf8a9de4cbfb3ac203df0c21a4d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/767ebaf8a9de4cbfb3ac203df0c21a4d 2024-12-05T07:42:01,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/767ebaf8a9de4cbfb3ac203df0c21a4d, entries=150, sequenceid=75, filesize=11.7 K 2024-12-05T07:42:01,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/0b2cef2b62ee48f28eaefc801d4cd845 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/0b2cef2b62ee48f28eaefc801d4cd845 2024-12-05T07:42:01,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/0b2cef2b62ee48f28eaefc801d4cd845, entries=150, sequenceid=75, filesize=11.7 K 2024-12-05T07:42:01,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 45734b659127c96dcd616039a555b19f in 871ms, sequenceid=75, compaction requested=false 2024-12-05T07:42:01,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:01,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T07:42:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:01,803 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T07:42:01,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/21afda56e7b849e2b12af1a87977af21 is 50, key is test_row_0/A:col10/1733384520866/Put/seqid=0 2024-12-05T07:42:01,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742242_1418 (size=12001) 2024-12-05T07:42:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T07:42:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:02,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T07:42:02,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384582141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:02,212 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/21afda56e7b849e2b12af1a87977af21 2024-12-05T07:42:02,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/78eb43e1762f4225b239b2acbd2aaea4 is 50, key is test_row_0/B:col10/1733384520866/Put/seqid=0 2024-12-05T07:42:02,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742243_1419 (size=12001) 2024-12-05T07:42:02,224 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/78eb43e1762f4225b239b2acbd2aaea4 2024-12-05T07:42:02,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/7ebbd3899a0e459c8ebd953295aa3658 is 50, key is test_row_0/C:col10/1733384520866/Put/seqid=0 2024-12-05T07:42:02,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742244_1420 (size=12001) 2024-12-05T07:42:02,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384582256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:02,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:02,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384582466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:02,633 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/7ebbd3899a0e459c8ebd953295aa3658 2024-12-05T07:42:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T07:42:02,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/21afda56e7b849e2b12af1a87977af21 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/21afda56e7b849e2b12af1a87977af21 2024-12-05T07:42:02,640 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/21afda56e7b849e2b12af1a87977af21, entries=150, sequenceid=89, filesize=11.7 K 2024-12-05T07:42:02,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/78eb43e1762f4225b239b2acbd2aaea4 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/78eb43e1762f4225b239b2acbd2aaea4 2024-12-05T07:42:02,643 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/78eb43e1762f4225b239b2acbd2aaea4, entries=150, sequenceid=89, filesize=11.7 K 2024-12-05T07:42:02,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/7ebbd3899a0e459c8ebd953295aa3658 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7ebbd3899a0e459c8ebd953295aa3658 2024-12-05T07:42:02,647 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7ebbd3899a0e459c8ebd953295aa3658, entries=150, sequenceid=89, filesize=11.7 K 2024-12-05T07:42:02,648 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 45734b659127c96dcd616039a555b19f in 845ms, sequenceid=89, compaction requested=true 2024-12-05T07:42:02,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:02,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:02,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-05T07:42:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-05T07:42:02,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-05T07:42:02,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1510 sec 2024-12-05T07:42:02,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.1550 sec 2024-12-05T07:42:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:02,679 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:02,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T07:42:02,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/2fb63c33b0734109bbdebb96210da1c3 is 50, key is test_row_0/A:col10/1733384522678/Put/seqid=0 2024-12-05T07:42:02,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742245_1421 (size=14341) 2024-12-05T07:42:02,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384582700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:02,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:02,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384582704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384582701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:02,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384582705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:02,712 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 45734b659127c96dcd616039a555b19f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:02,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384582776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:02,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384582815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:02,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384582816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:02,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384582816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:02,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:03,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384583025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:03,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384583025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:03,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384583026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:03,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/2fb63c33b0734109bbdebb96210da1c3 2024-12-05T07:42:03,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/fe1c91359a6d42588e440e231decfbae is 50, key is test_row_0/B:col10/1733384522678/Put/seqid=0 2024-12-05T07:42:03,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742246_1422 (size=12001) 2024-12-05T07:42:03,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384583296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:03,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:03,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:03,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384583336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384583336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:03,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384583336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:03,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/fe1c91359a6d42588e440e231decfbae 2024-12-05T07:42:03,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/f8a18d9f5f0c44aeb067571c70066666 is 50, key is test_row_0/C:col10/1733384522678/Put/seqid=0 2024-12-05T07:42:03,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742247_1423 (size=12001) 2024-12-05T07:42:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T07:42:03,646 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:03,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-05T07:42:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T07:42:03,648 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:03,648 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:03,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T07:42:03,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-05T07:42:03,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:03,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:03,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:03,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:03,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:03,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384583856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:03,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384583856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:03,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384583856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:03,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:03,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/f8a18d9f5f0c44aeb067571c70066666 2024-12-05T07:42:03,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/2fb63c33b0734109bbdebb96210da1c3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/2fb63c33b0734109bbdebb96210da1c3 2024-12-05T07:42:03,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/2fb63c33b0734109bbdebb96210da1c3, entries=200, sequenceid=114, filesize=14.0 K 2024-12-05T07:42:03,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/fe1c91359a6d42588e440e231decfbae as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/fe1c91359a6d42588e440e231decfbae 2024-12-05T07:42:03,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/fe1c91359a6d42588e440e231decfbae, entries=150, sequenceid=114, filesize=11.7 K 2024-12-05T07:42:03,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/f8a18d9f5f0c44aeb067571c70066666 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f8a18d9f5f0c44aeb067571c70066666 2024-12-05T07:42:03,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f8a18d9f5f0c44aeb067571c70066666, entries=150, sequenceid=114, filesize=11.7 K 2024-12-05T07:42:03,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 45734b659127c96dcd616039a555b19f in 1242ms, sequenceid=114, compaction requested=true 2024-12-05T07:42:03,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:03,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:03,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:03,921 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:42:03,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:03,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:03,921 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:42:03,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:03,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:03,922 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:42:03,922 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:42:03,922 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/A is initiating minor compaction (all files) 2024-12-05T07:42:03,922 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/B is initiating minor compaction (all files) 2024-12-05T07:42:03,922 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/A in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:03,922 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/B in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:03,922 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5c14fdac638746ccb547ca54125145d1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/687e677a709a4c76a8da017701b2012c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/21afda56e7b849e2b12af1a87977af21, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/2fb63c33b0734109bbdebb96210da1c3] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=51.5 K 2024-12-05T07:42:03,922 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/859e7adbcc184490a22677dd447b308f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/767ebaf8a9de4cbfb3ac203df0c21a4d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/78eb43e1762f4225b239b2acbd2aaea4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/fe1c91359a6d42588e440e231decfbae] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=47.0 K 2024-12-05T07:42:03,922 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c14fdac638746ccb547ca54125145d1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733384518504 2024-12-05T07:42:03,922 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 859e7adbcc184490a22677dd447b308f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733384518504 2024-12-05T07:42:03,922 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 687e677a709a4c76a8da017701b2012c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733384519636 2024-12-05T07:42:03,922 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 767ebaf8a9de4cbfb3ac203df0c21a4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733384519636 2024-12-05T07:42:03,923 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21afda56e7b849e2b12af1a87977af21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733384520846 2024-12-05T07:42:03,923 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 78eb43e1762f4225b239b2acbd2aaea4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733384520846 2024-12-05T07:42:03,923 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb63c33b0734109bbdebb96210da1c3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733384522114 2024-12-05T07:42:03,923 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting fe1c91359a6d42588e440e231decfbae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733384522114 2024-12-05T07:42:03,928 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:03,929 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/48f038d3e9c344088030f207b514233b is 50, key is test_row_0/B:col10/1733384522678/Put/seqid=0 2024-12-05T07:42:03,929 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#A#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:03,929 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/b2b98a63360d47c99a3b41bae281cfbb is 50, key is test_row_0/A:col10/1733384522678/Put/seqid=0 2024-12-05T07:42:03,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742249_1425 (size=12241) 2024-12-05T07:42:03,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742248_1424 (size=12241) 2024-12-05T07:42:03,940 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/48f038d3e9c344088030f207b514233b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/48f038d3e9c344088030f207b514233b 2024-12-05T07:42:03,942 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45734b659127c96dcd616039a555b19f/B of 45734b659127c96dcd616039a555b19f into 48f038d3e9c344088030f207b514233b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:03,942 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:03,942 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/B, priority=12, startTime=1733384523921; duration=0sec 2024-12-05T07:42:03,942 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:03,943 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:B 2024-12-05T07:42:03,943 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:42:03,944 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:42:03,944 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/C is initiating minor compaction (all files) 2024-12-05T07:42:03,944 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/C in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:03,944 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/5b09548714f0468cbc87d77f2572ee62, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/0b2cef2b62ee48f28eaefc801d4cd845, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7ebbd3899a0e459c8ebd953295aa3658, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f8a18d9f5f0c44aeb067571c70066666] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=47.0 K 2024-12-05T07:42:03,944 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b09548714f0468cbc87d77f2572ee62, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733384518504 2024-12-05T07:42:03,947 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b2cef2b62ee48f28eaefc801d4cd845, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1733384519636 2024-12-05T07:42:03,947 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ebbd3899a0e459c8ebd953295aa3658, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1733384520846 2024-12-05T07:42:03,947 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f8a18d9f5f0c44aeb067571c70066666, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733384522114 2024-12-05T07:42:03,952 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#C#compaction#355 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:03,953 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/a1e713fe62ac4331a107e0a32b18b3f1 is 50, key is test_row_0/C:col10/1733384522678/Put/seqid=0 2024-12-05T07:42:03,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-05T07:42:03,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:03,953 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:42:03,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742250_1426 (size=12241) 2024-12-05T07:42:03,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/d6f4bed34d134158bc896205d0a4c4c6 is 50, key is test_row_0/A:col10/1733384522699/Put/seqid=0 2024-12-05T07:42:03,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742251_1427 (size=12001) 2024-12-05T07:42:03,960 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/d6f4bed34d134158bc896205d0a4c4c6 2024-12-05T07:42:03,961 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/a1e713fe62ac4331a107e0a32b18b3f1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a1e713fe62ac4331a107e0a32b18b3f1 2024-12-05T07:42:03,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/05c51842745b40c989f65b474a329bca is 50, key is test_row_0/B:col10/1733384522699/Put/seqid=0 2024-12-05T07:42:03,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T07:42:03,966 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45734b659127c96dcd616039a555b19f/C of 45734b659127c96dcd616039a555b19f into a1e713fe62ac4331a107e0a32b18b3f1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:03,967 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:03,967 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/C, priority=12, startTime=1733384523921; duration=0sec 2024-12-05T07:42:03,967 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:03,967 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:C 2024-12-05T07:42:03,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742252_1428 (size=12001) 2024-12-05T07:42:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T07:42:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:04,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:04,340 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/b2b98a63360d47c99a3b41bae281cfbb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/b2b98a63360d47c99a3b41bae281cfbb 2024-12-05T07:42:04,344 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45734b659127c96dcd616039a555b19f/A of 45734b659127c96dcd616039a555b19f into b2b98a63360d47c99a3b41bae281cfbb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:04,345 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:04,345 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/A, priority=12, startTime=1733384523921; duration=0sec 2024-12-05T07:42:04,345 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:04,345 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:A 2024-12-05T07:42:04,373 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/05c51842745b40c989f65b474a329bca 2024-12-05T07:42:04,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/263867a76de440dc8ee0f6d017dbce16 is 50, key is test_row_0/C:col10/1733384522699/Put/seqid=0 2024-12-05T07:42:04,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742253_1429 (size=12001) 2024-12-05T07:42:04,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384584397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:04,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384584506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:04,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384584715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:04,783 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/263867a76de440dc8ee0f6d017dbce16 2024-12-05T07:42:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T07:42:04,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/d6f4bed34d134158bc896205d0a4c4c6 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d6f4bed34d134158bc896205d0a4c4c6 2024-12-05T07:42:04,789 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d6f4bed34d134158bc896205d0a4c4c6, entries=150, sequenceid=126, filesize=11.7 K 2024-12-05T07:42:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/05c51842745b40c989f65b474a329bca as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/05c51842745b40c989f65b474a329bca 2024-12-05T07:42:04,796 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/05c51842745b40c989f65b474a329bca, entries=150, sequenceid=126, filesize=11.7 K 2024-12-05T07:42:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/263867a76de440dc8ee0f6d017dbce16 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/263867a76de440dc8ee0f6d017dbce16 2024-12-05T07:42:04,800 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/263867a76de440dc8ee0f6d017dbce16, entries=150, sequenceid=126, filesize=11.7 K 2024-12-05T07:42:04,800 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 45734b659127c96dcd616039a555b19f in 847ms, sequenceid=126, compaction requested=false 2024-12-05T07:42:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-05T07:42:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-05T07:42:04,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-05T07:42:04,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1530 sec 2024-12-05T07:42:04,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.1560 sec 2024-12-05T07:42:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:04,869 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:04,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T07:42:04,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:04,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384584880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:04,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384584881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:04,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384584884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:04,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/bc0b4949075e42e4b7e765ea86835c98 is 50, key is test_row_0/A:col10/1733384524868/Put/seqid=0 2024-12-05T07:42:04,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742254_1430 (size=14541) 2024-12-05T07:42:04,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384584996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:05,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384584996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:05,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384584996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:05,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384585025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:05,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384585206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:05,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384585206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:05,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384585206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:05,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/bc0b4949075e42e4b7e765ea86835c98 2024-12-05T07:42:05,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/8b907d2198da4e9fbb8dc28df428eb2e is 50, key is test_row_0/B:col10/1733384524868/Put/seqid=0 2024-12-05T07:42:05,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742255_1431 (size=12151) 2024-12-05T07:42:05,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384585516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384585516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384585516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:05,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384585536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:05,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/8b907d2198da4e9fbb8dc28df428eb2e 2024-12-05T07:42:05,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/37e1694d3358453488ae43d26f5c3f7e is 50, key is test_row_0/C:col10/1733384524868/Put/seqid=0 2024-12-05T07:42:05,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742256_1432 (size=12151) 2024-12-05T07:42:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T07:42:05,806 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:05,807 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-05T07:42:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T07:42:05,808 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:05,809 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:05,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T07:42:05,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T07:42:05,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:05,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:05,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:06,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384586026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:06,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384586026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384586026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:06,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T07:42:06,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:06,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:06,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:06,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:06,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:06,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/37e1694d3358453488ae43d26f5c3f7e 2024-12-05T07:42:06,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/bc0b4949075e42e4b7e765ea86835c98 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/bc0b4949075e42e4b7e765ea86835c98 2024-12-05T07:42:06,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/bc0b4949075e42e4b7e765ea86835c98, entries=200, sequenceid=154, filesize=14.2 K 2024-12-05T07:42:06,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/8b907d2198da4e9fbb8dc28df428eb2e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8b907d2198da4e9fbb8dc28df428eb2e 2024-12-05T07:42:06,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8b907d2198da4e9fbb8dc28df428eb2e, entries=150, sequenceid=154, filesize=11.9 K 2024-12-05T07:42:06,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/37e1694d3358453488ae43d26f5c3f7e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/37e1694d3358453488ae43d26f5c3f7e 2024-12-05T07:42:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T07:42:06,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/37e1694d3358453488ae43d26f5c3f7e, entries=150, sequenceid=154, filesize=11.9 K 2024-12-05T07:42:06,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 45734b659127c96dcd616039a555b19f in 1259ms, sequenceid=154, compaction requested=true 2024-12-05T07:42:06,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:06,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:06,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:06,129 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:06,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:06,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:06,129 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:06,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:06,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:06,129 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:06,129 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:06,129 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/A is initiating minor compaction (all files) 2024-12-05T07:42:06,129 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/B is initiating minor compaction (all files) 2024-12-05T07:42:06,129 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/A in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:06,129 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/B in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:06,130 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/b2b98a63360d47c99a3b41bae281cfbb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d6f4bed34d134158bc896205d0a4c4c6, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/bc0b4949075e42e4b7e765ea86835c98] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=37.9 K 2024-12-05T07:42:06,130 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/48f038d3e9c344088030f207b514233b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/05c51842745b40c989f65b474a329bca, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8b907d2198da4e9fbb8dc28df428eb2e] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=35.5 K 2024-12-05T07:42:06,130 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 48f038d3e9c344088030f207b514233b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733384522114 2024-12-05T07:42:06,130 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2b98a63360d47c99a3b41bae281cfbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733384522114 2024-12-05T07:42:06,130 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 05c51842745b40c989f65b474a329bca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733384522692 2024-12-05T07:42:06,130 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6f4bed34d134158bc896205d0a4c4c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733384522692 2024-12-05T07:42:06,130 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc0b4949075e42e4b7e765ea86835c98, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733384524381 2024-12-05T07:42:06,130 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b907d2198da4e9fbb8dc28df428eb2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733384524381 2024-12-05T07:42:06,135 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#A#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:06,136 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/329bb74e567446779474f2bb6367365b is 50, key is test_row_0/A:col10/1733384524868/Put/seqid=0 2024-12-05T07:42:06,136 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:06,137 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/49ead8faa2994789bfe18ad740818ae1 is 50, key is test_row_0/B:col10/1733384524868/Put/seqid=0 2024-12-05T07:42:06,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742258_1434 (size=12493) 2024-12-05T07:42:06,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742257_1433 (size=12493) 2024-12-05T07:42:06,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T07:42:06,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:06,266 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-05T07:42:06,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/4c406b5e58f94c27bfe1f93f1b57fa71 is 50, key is test_row_0/A:col10/1733384524883/Put/seqid=0 2024-12-05T07:42:06,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742259_1435 (size=12151) 2024-12-05T07:42:06,274 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/4c406b5e58f94c27bfe1f93f1b57fa71 2024-12-05T07:42:06,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/cd21986985d84db986fe742b00ce3f69 is 50, key is test_row_0/B:col10/1733384524883/Put/seqid=0 2024-12-05T07:42:06,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742260_1436 (size=12151) 2024-12-05T07:42:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T07:42:06,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:06,555 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/329bb74e567446779474f2bb6367365b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/329bb74e567446779474f2bb6367365b 2024-12-05T07:42:06,555 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/49ead8faa2994789bfe18ad740818ae1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/49ead8faa2994789bfe18ad740818ae1 2024-12-05T07:42:06,560 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/B of 45734b659127c96dcd616039a555b19f into 49ead8faa2994789bfe18ad740818ae1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:06,560 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/A of 45734b659127c96dcd616039a555b19f into 329bb74e567446779474f2bb6367365b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:06,560 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:06,560 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:06,560 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/B, priority=13, startTime=1733384526129; duration=0sec 2024-12-05T07:42:06,560 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/A, priority=13, startTime=1733384526129; duration=0sec 2024-12-05T07:42:06,560 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:06,560 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:B 2024-12-05T07:42:06,560 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:06,560 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:A 2024-12-05T07:42:06,560 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:06,561 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:06,561 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/C is initiating minor compaction (all files) 2024-12-05T07:42:06,562 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/C in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:06,562 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a1e713fe62ac4331a107e0a32b18b3f1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/263867a76de440dc8ee0f6d017dbce16, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/37e1694d3358453488ae43d26f5c3f7e] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=35.5 K 2024-12-05T07:42:06,562 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a1e713fe62ac4331a107e0a32b18b3f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1733384522114 2024-12-05T07:42:06,562 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 263867a76de440dc8ee0f6d017dbce16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1733384522692 2024-12-05T07:42:06,562 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 37e1694d3358453488ae43d26f5c3f7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733384524381 2024-12-05T07:42:06,568 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#C#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:06,569 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/f753451c958045bcb22b29abd8d8476f is 50, key is test_row_0/C:col10/1733384524868/Put/seqid=0 2024-12-05T07:42:06,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742261_1437 (size=12493) 2024-12-05T07:42:06,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384586669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:06,683 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/cd21986985d84db986fe742b00ce3f69 2024-12-05T07:42:06,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/2a4a4feb75e94abf9de7e32dc60fd254 is 50, key is test_row_0/C:col10/1733384524883/Put/seqid=0 2024-12-05T07:42:06,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742262_1438 (size=12151) 2024-12-05T07:42:06,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:06,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384586736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:06,740 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 45734b659127c96dcd616039a555b19f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:06,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384586785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T07:42:06,976 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/f753451c958045bcb22b29abd8d8476f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f753451c958045bcb22b29abd8d8476f 2024-12-05T07:42:06,979 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/C of 45734b659127c96dcd616039a555b19f into f753451c958045bcb22b29abd8d8476f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:06,979 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:06,980 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/C, priority=13, startTime=1733384526129; duration=0sec 2024-12-05T07:42:06,980 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:06,980 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:C 2024-12-05T07:42:06,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:06,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384586996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:06,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:07,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:07,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384587035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:07,039 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in 45734b659127c96dcd616039a555b19f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:07,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384587046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:07,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384587046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:07,049 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in 45734b659127c96dcd616039a555b19f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:07,049 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in 45734b659127c96dcd616039a555b19f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:07,094 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/2a4a4feb75e94abf9de7e32dc60fd254 2024-12-05T07:42:07,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/4c406b5e58f94c27bfe1f93f1b57fa71 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/4c406b5e58f94c27bfe1f93f1b57fa71 2024-12-05T07:42:07,102 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/4c406b5e58f94c27bfe1f93f1b57fa71, entries=150, sequenceid=164, filesize=11.9 K 2024-12-05T07:42:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/cd21986985d84db986fe742b00ce3f69 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cd21986985d84db986fe742b00ce3f69 2024-12-05T07:42:07,105 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cd21986985d84db986fe742b00ce3f69, entries=150, sequenceid=164, filesize=11.9 K 2024-12-05T07:42:07,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/2a4a4feb75e94abf9de7e32dc60fd254 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2a4a4feb75e94abf9de7e32dc60fd254 2024-12-05T07:42:07,108 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2a4a4feb75e94abf9de7e32dc60fd254, entries=150, sequenceid=164, filesize=11.9 K 2024-12-05T07:42:07,109 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 45734b659127c96dcd616039a555b19f in 844ms, sequenceid=164, compaction requested=false 2024-12-05T07:42:07,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:07,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:07,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-05T07:42:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-05T07:42:07,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-05T07:42:07,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3010 sec 2024-12-05T07:42:07,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.3050 sec 2024-12-05T07:42:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:07,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-05T07:42:07,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/d4c74279041948e984bc543fd0fc52bc is 50, key is test_row_0/A:col10/1733384527306/Put/seqid=0 2024-12-05T07:42:07,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742263_1439 (size=14541) 2024-12-05T07:42:07,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384587336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384587446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:07,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384587656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:07,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/d4c74279041948e984bc543fd0fc52bc 2024-12-05T07:42:07,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/2b05b360fb5945a69523d9ae792dba6d is 50, key is test_row_0/B:col10/1733384527306/Put/seqid=0 2024-12-05T07:42:07,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742264_1440 (size=12151) 2024-12-05T07:42:07,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T07:42:07,956 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:07,957 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:07,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-05T07:42:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T07:42:07,958 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:07,959 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:07,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:07,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384587965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T07:42:08,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T07:42:08,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:08,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:08,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:08,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:08,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:08,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/2b05b360fb5945a69523d9ae792dba6d 2024-12-05T07:42:08,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/f93e8facd15340bea9417b55d799e7d1 is 50, key is test_row_0/C:col10/1733384527306/Put/seqid=0 2024-12-05T07:42:08,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742265_1441 (size=12151) 2024-12-05T07:42:08,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T07:42:08,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:08,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:08,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:08,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:08,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T07:42:08,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T07:42:08,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:08,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:08,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:08,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:08,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:08,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:08,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384588476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:08,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:08,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/f93e8facd15340bea9417b55d799e7d1 2024-12-05T07:42:08,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/d4c74279041948e984bc543fd0fc52bc as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d4c74279041948e984bc543fd0fc52bc 2024-12-05T07:42:08,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d4c74279041948e984bc543fd0fc52bc, entries=200, sequenceid=194, filesize=14.2 K 2024-12-05T07:42:08,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/2b05b360fb5945a69523d9ae792dba6d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2b05b360fb5945a69523d9ae792dba6d 2024-12-05T07:42:08,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2b05b360fb5945a69523d9ae792dba6d, entries=150, sequenceid=194, filesize=11.9 K 2024-12-05T07:42:08,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/f93e8facd15340bea9417b55d799e7d1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f93e8facd15340bea9417b55d799e7d1 2024-12-05T07:42:08,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f93e8facd15340bea9417b55d799e7d1, entries=150, sequenceid=194, filesize=11.9 K 2024-12-05T07:42:08,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 45734b659127c96dcd616039a555b19f in 1240ms, sequenceid=194, compaction requested=true 2024-12-05T07:42:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:08,547 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:08,547 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:08,548 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:08,548 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:08,548 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/B is initiating minor compaction (all files) 2024-12-05T07:42:08,548 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/A is initiating minor compaction (all files) 2024-12-05T07:42:08,548 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/B in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:08,548 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/A in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:08,548 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/49ead8faa2994789bfe18ad740818ae1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cd21986985d84db986fe742b00ce3f69, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2b05b360fb5945a69523d9ae792dba6d] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=35.9 K 2024-12-05T07:42:08,548 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/329bb74e567446779474f2bb6367365b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/4c406b5e58f94c27bfe1f93f1b57fa71, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d4c74279041948e984bc543fd0fc52bc] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=38.3 K 2024-12-05T07:42:08,548 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 49ead8faa2994789bfe18ad740818ae1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733384524381 2024-12-05T07:42:08,548 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 329bb74e567446779474f2bb6367365b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733384524381 2024-12-05T07:42:08,549 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting cd21986985d84db986fe742b00ce3f69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733384524873 2024-12-05T07:42:08,549 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c406b5e58f94c27bfe1f93f1b57fa71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733384524873 2024-12-05T07:42:08,549 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b05b360fb5945a69523d9ae792dba6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733384526657 2024-12-05T07:42:08,549 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4c74279041948e984bc543fd0fc52bc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733384526622 2024-12-05T07:42:08,555 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#A#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:08,555 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/476663e650e44fef9560fc276b045ef2 is 50, key is test_row_0/A:col10/1733384527306/Put/seqid=0 2024-12-05T07:42:08,557 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:08,558 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/800441b784364514820cad8f92f9f034 is 50, key is test_row_0/B:col10/1733384527306/Put/seqid=0 2024-12-05T07:42:08,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742266_1442 (size=12595) 2024-12-05T07:42:08,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742267_1443 (size=12595) 2024-12-05T07:42:08,565 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/476663e650e44fef9560fc276b045ef2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/476663e650e44fef9560fc276b045ef2 2024-12-05T07:42:08,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T07:42:08,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:08,570 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-12-05T07:42:08,570 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/A of 45734b659127c96dcd616039a555b19f into 476663e650e44fef9560fc276b045ef2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:08,570 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:08,570 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/A, priority=13, startTime=1733384528547; duration=0sec 2024-12-05T07:42:08,570 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:08,570 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:A 2024-12-05T07:42:08,570 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:08,571 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:08,571 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/C is initiating minor compaction (all files) 2024-12-05T07:42:08,571 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/C in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:08,571 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f753451c958045bcb22b29abd8d8476f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2a4a4feb75e94abf9de7e32dc60fd254, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f93e8facd15340bea9417b55d799e7d1] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=35.9 K 2024-12-05T07:42:08,572 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f753451c958045bcb22b29abd8d8476f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733384524381 2024-12-05T07:42:08,572 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a4a4feb75e94abf9de7e32dc60fd254, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733384524873 2024-12-05T07:42:08,574 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f93e8facd15340bea9417b55d799e7d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733384526657 2024-12-05T07:42:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/f25176863770402c906bddc7f3170871 is 50, key is test_row_0/A:col10/1733384527330/Put/seqid=0 2024-12-05T07:42:08,581 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:08,581 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/2231745005204b50a528e6572573a7a2 is 50, key is test_row_0/C:col10/1733384527306/Put/seqid=0 2024-12-05T07:42:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T07:42:08,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742268_1444 (size=12151) 2024-12-05T07:42:08,588 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/f25176863770402c906bddc7f3170871 2024-12-05T07:42:08,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/0bfe1ec0801c49229d650b9496c0cbd7 is 50, key is test_row_0/B:col10/1733384527330/Put/seqid=0 2024-12-05T07:42:08,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742269_1445 (size=12595) 2024-12-05T07:42:08,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742270_1446 (size=12151) 2024-12-05T07:42:08,598 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/2231745005204b50a528e6572573a7a2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2231745005204b50a528e6572573a7a2 2024-12-05T07:42:08,602 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/C of 45734b659127c96dcd616039a555b19f into 2231745005204b50a528e6572573a7a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:08,602 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:08,602 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/C, priority=13, startTime=1733384528547; duration=0sec 2024-12-05T07:42:08,602 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:08,602 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:C 2024-12-05T07:42:08,965 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/800441b784364514820cad8f92f9f034 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/800441b784364514820cad8f92f9f034 2024-12-05T07:42:08,969 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/B of 45734b659127c96dcd616039a555b19f into 800441b784364514820cad8f92f9f034(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:08,969 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:08,969 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/B, priority=13, startTime=1733384528547; duration=0sec 2024-12-05T07:42:08,969 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:08,969 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:B 2024-12-05T07:42:08,998 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/0bfe1ec0801c49229d650b9496c0cbd7 2024-12-05T07:42:09,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/6a47bd24af4f4ee4b0ccfcf878f93597 is 50, key is test_row_0/C:col10/1733384527330/Put/seqid=0 2024-12-05T07:42:09,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742271_1447 (size=12151) 2024-12-05T07:42:09,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T07:42:09,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384589107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384589108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384589109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384589216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384589216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384589226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,408 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/6a47bd24af4f4ee4b0ccfcf878f93597 2024-12-05T07:42:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/f25176863770402c906bddc7f3170871 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/f25176863770402c906bddc7f3170871 2024-12-05T07:42:09,414 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/f25176863770402c906bddc7f3170871, entries=150, sequenceid=204, filesize=11.9 K 2024-12-05T07:42:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/0bfe1ec0801c49229d650b9496c0cbd7 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/0bfe1ec0801c49229d650b9496c0cbd7 2024-12-05T07:42:09,417 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/0bfe1ec0801c49229d650b9496c0cbd7, entries=150, sequenceid=204, filesize=11.9 K 2024-12-05T07:42:09,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/6a47bd24af4f4ee4b0ccfcf878f93597 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6a47bd24af4f4ee4b0ccfcf878f93597 2024-12-05T07:42:09,420 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6a47bd24af4f4ee4b0ccfcf878f93597, entries=150, sequenceid=204, filesize=11.9 K 2024-12-05T07:42:09,421 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 45734b659127c96dcd616039a555b19f in 852ms, sequenceid=204, compaction requested=false 2024-12-05T07:42:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-05T07:42:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-05T07:42:09,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-05T07:42:09,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4630 sec 2024-12-05T07:42:09,424 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4660 sec 2024-12-05T07:42:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:09,429 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:09,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-12-05T07:42:09,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/8ab369edb81a4fbba816bb1fa3499e95 is 50, key is test_row_0/A:col10/1733384529108/Put/seqid=0 2024-12-05T07:42:09,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742272_1448 (size=16931) 2024-12-05T07:42:09,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/8ab369edb81a4fbba816bb1fa3499e95 2024-12-05T07:42:09,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/c8899b2e7d38453da5419ffdd9ef18c6 is 50, key is test_row_0/B:col10/1733384529108/Put/seqid=0 2024-12-05T07:42:09,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384589439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384589440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742273_1449 (size=12151) 2024-12-05T07:42:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/c8899b2e7d38453da5419ffdd9ef18c6 2024-12-05T07:42:09,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/a48328e8d7c44e4f8d5db859e16c438a is 50, key is test_row_0/C:col10/1733384529108/Put/seqid=0 2024-12-05T07:42:09,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384589447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742274_1450 (size=12151) 2024-12-05T07:42:09,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/a48328e8d7c44e4f8d5db859e16c438a 2024-12-05T07:42:09,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/8ab369edb81a4fbba816bb1fa3499e95 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/8ab369edb81a4fbba816bb1fa3499e95 2024-12-05T07:42:09,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/8ab369edb81a4fbba816bb1fa3499e95, entries=250, sequenceid=235, filesize=16.5 K 2024-12-05T07:42:09,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/c8899b2e7d38453da5419ffdd9ef18c6 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/c8899b2e7d38453da5419ffdd9ef18c6 2024-12-05T07:42:09,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/c8899b2e7d38453da5419ffdd9ef18c6, entries=150, sequenceid=235, filesize=11.9 K 2024-12-05T07:42:09,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/a48328e8d7c44e4f8d5db859e16c438a as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a48328e8d7c44e4f8d5db859e16c438a 2024-12-05T07:42:09,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a48328e8d7c44e4f8d5db859e16c438a, entries=150, sequenceid=235, filesize=11.9 K 2024-12-05T07:42:09,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 45734b659127c96dcd616039a555b19f in 40ms, sequenceid=235, compaction requested=true 2024-12-05T07:42:09,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:09,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:09,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:09,469 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:09,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:09,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:09,469 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:09,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:09,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:09,470 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:09,470 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:09,471 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/A is initiating minor compaction (all files) 2024-12-05T07:42:09,471 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/B is initiating minor compaction (all files) 2024-12-05T07:42:09,471 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/B in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:09,471 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/A in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:09,471 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/476663e650e44fef9560fc276b045ef2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/f25176863770402c906bddc7f3170871, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/8ab369edb81a4fbba816bb1fa3499e95] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=40.7 K 2024-12-05T07:42:09,471 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/800441b784364514820cad8f92f9f034, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/0bfe1ec0801c49229d650b9496c0cbd7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/c8899b2e7d38453da5419ffdd9ef18c6] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=36.0 K 2024-12-05T07:42:09,471 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 800441b784364514820cad8f92f9f034, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733384526657 2024-12-05T07:42:09,471 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 476663e650e44fef9560fc276b045ef2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733384526657 2024-12-05T07:42:09,473 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f25176863770402c906bddc7f3170871, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733384527319 2024-12-05T07:42:09,473 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bfe1ec0801c49229d650b9496c0cbd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733384527319 2024-12-05T07:42:09,473 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab369edb81a4fbba816bb1fa3499e95, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733384529106 2024-12-05T07:42:09,473 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting c8899b2e7d38453da5419ffdd9ef18c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733384529107 2024-12-05T07:42:09,479 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#B#compaction#381 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:09,480 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#A#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:09,480 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/8f6776ae5fb4490f971883bc9750e054 is 50, key is test_row_0/B:col10/1733384529108/Put/seqid=0 2024-12-05T07:42:09,480 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/713f45a726e747ea9675141d7e0b9e89 is 50, key is test_row_0/A:col10/1733384529108/Put/seqid=0 2024-12-05T07:42:09,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742275_1451 (size=12697) 2024-12-05T07:42:09,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742276_1452 (size=12697) 2024-12-05T07:42:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:09,513 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:09,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:42:09,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/81352c29392c4dbdaa4de8591669909d is 50, key is test_row_0/A:col10/1733384529503/Put/seqid=0 2024-12-05T07:42:09,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742277_1453 (size=14541) 2024-12-05T07:42:09,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384589584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384589584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384589592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384589593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384589695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384589696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384589706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384589706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384589905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384589906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/81352c29392c4dbdaa4de8591669909d 2024-12-05T07:42:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384589916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:09,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384589916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:09,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:09,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:09,943 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/8f6776ae5fb4490f971883bc9750e054 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8f6776ae5fb4490f971883bc9750e054 2024-12-05T07:42:09,943 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/713f45a726e747ea9675141d7e0b9e89 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/713f45a726e747ea9675141d7e0b9e89 2024-12-05T07:42:09,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/2a7e5538f8d04f4b9b9c8827809f51c4 is 50, key is test_row_0/B:col10/1733384529503/Put/seqid=0 2024-12-05T07:42:09,950 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/A of 45734b659127c96dcd616039a555b19f into 713f45a726e747ea9675141d7e0b9e89(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:09,950 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:09,950 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/A, priority=13, startTime=1733384529469; duration=0sec 2024-12-05T07:42:09,950 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:09,950 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:A 2024-12-05T07:42:09,950 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:09,950 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:09,951 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/C is initiating minor compaction (all files) 2024-12-05T07:42:09,951 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/C in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:09,951 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2231745005204b50a528e6572573a7a2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6a47bd24af4f4ee4b0ccfcf878f93597, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a48328e8d7c44e4f8d5db859e16c438a] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=36.0 K 2024-12-05T07:42:09,951 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2231745005204b50a528e6572573a7a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733384526657 2024-12-05T07:42:09,951 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a47bd24af4f4ee4b0ccfcf878f93597, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733384527319 2024-12-05T07:42:09,952 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a48328e8d7c44e4f8d5db859e16c438a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733384529107 2024-12-05T07:42:09,953 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/B of 45734b659127c96dcd616039a555b19f into 8f6776ae5fb4490f971883bc9750e054(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:09,953 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:09,953 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/B, priority=13, startTime=1733384529469; duration=0sec 2024-12-05T07:42:09,953 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:09,953 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:B 2024-12-05T07:42:09,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742278_1454 (size=12151) 2024-12-05T07:42:09,965 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#C#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:09,965 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/46c9c5567b9f441d819e5895ef478d5e is 50, key is test_row_0/C:col10/1733384529108/Put/seqid=0 2024-12-05T07:42:09,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742279_1455 (size=12697) 2024-12-05T07:42:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T07:42:10,116 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:10,116 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-05T07:42:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T07:42:10,118 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:10,118 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:10,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:10,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384590216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384590216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T07:42:10,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384590225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384590226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:10,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T07:42:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:10,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/2a7e5538f8d04f4b9b9c8827809f51c4 2024-12-05T07:42:10,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/7f6ad06b766f46578b3df12252293a5d is 50, key is test_row_0/C:col10/1733384529503/Put/seqid=0 2024-12-05T07:42:10,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742280_1456 (size=12151) 2024-12-05T07:42:10,373 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/46c9c5567b9f441d819e5895ef478d5e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/46c9c5567b9f441d819e5895ef478d5e 2024-12-05T07:42:10,376 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/C of 45734b659127c96dcd616039a555b19f into 46c9c5567b9f441d819e5895ef478d5e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:10,376 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:10,376 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/C, priority=13, startTime=1733384529469; duration=0sec 2024-12-05T07:42:10,376 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:10,376 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:C 2024-12-05T07:42:10,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T07:42:10,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:10,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:10,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:10,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T07:42:10,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T07:42:10,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:10,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:10,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:10,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T07:42:10,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:10,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:10,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:10,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384590725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:10,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:10,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384590736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384590736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T07:42:10,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384590746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:10,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/7f6ad06b766f46578b3df12252293a5d 2024-12-05T07:42:10,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/81352c29392c4dbdaa4de8591669909d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/81352c29392c4dbdaa4de8591669909d 2024-12-05T07:42:10,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/81352c29392c4dbdaa4de8591669909d, entries=200, sequenceid=246, filesize=14.2 K 2024-12-05T07:42:10,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/2a7e5538f8d04f4b9b9c8827809f51c4 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2a7e5538f8d04f4b9b9c8827809f51c4 2024-12-05T07:42:10,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2a7e5538f8d04f4b9b9c8827809f51c4, entries=150, sequenceid=246, filesize=11.9 K 2024-12-05T07:42:10,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/7f6ad06b766f46578b3df12252293a5d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7f6ad06b766f46578b3df12252293a5d 2024-12-05T07:42:10,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7f6ad06b766f46578b3df12252293a5d, entries=150, sequenceid=246, filesize=11.9 K 2024-12-05T07:42:10,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 45734b659127c96dcd616039a555b19f in 1269ms, sequenceid=246, compaction requested=false 2024-12-05T07:42:10,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:10,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T07:42:10,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:10,881 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-12-05T07:42:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/db4be35ed30a405aa00b7abfb400e908 is 50, key is test_row_0/A:col10/1733384529582/Put/seqid=0 2024-12-05T07:42:10,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742281_1457 (size=12301) 2024-12-05T07:42:10,888 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/db4be35ed30a405aa00b7abfb400e908 2024-12-05T07:42:10,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/5f52eaf121ba4bd8b027d8b49c2375a8 is 50, key is test_row_0/B:col10/1733384529582/Put/seqid=0 2024-12-05T07:42:10,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742282_1458 (size=12301) 2024-12-05T07:42:11,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T07:42:11,299 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/5f52eaf121ba4bd8b027d8b49c2375a8 2024-12-05T07:42:11,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/6bf37804ca2541dc931b3d0c2af3c10e is 50, key is test_row_0/C:col10/1733384529582/Put/seqid=0 2024-12-05T07:42:11,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742283_1459 (size=12301) 2024-12-05T07:42:11,709 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/6bf37804ca2541dc931b3d0c2af3c10e 2024-12-05T07:42:11,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/db4be35ed30a405aa00b7abfb400e908 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/db4be35ed30a405aa00b7abfb400e908 2024-12-05T07:42:11,715 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/db4be35ed30a405aa00b7abfb400e908, entries=150, sequenceid=274, filesize=12.0 K 2024-12-05T07:42:11,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/5f52eaf121ba4bd8b027d8b49c2375a8 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/5f52eaf121ba4bd8b027d8b49c2375a8 2024-12-05T07:42:11,718 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/5f52eaf121ba4bd8b027d8b49c2375a8, entries=150, sequenceid=274, filesize=12.0 K 2024-12-05T07:42:11,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/6bf37804ca2541dc931b3d0c2af3c10e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6bf37804ca2541dc931b3d0c2af3c10e 2024-12-05T07:42:11,722 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6bf37804ca2541dc931b3d0c2af3c10e, entries=150, sequenceid=274, filesize=12.0 K 2024-12-05T07:42:11,723 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 45734b659127c96dcd616039a555b19f in 842ms, sequenceid=274, compaction requested=true 2024-12-05T07:42:11,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:11,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:11,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-05T07:42:11,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-05T07:42:11,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-05T07:42:11,726 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6060 sec 2024-12-05T07:42:11,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.6100 sec 2024-12-05T07:42:11,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:11,765 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:11,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-12-05T07:42:11,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/58b08705b3094595aaf3245943b1226c is 50, key is test_row_0/A:col10/1733384531764/Put/seqid=0 2024-12-05T07:42:11,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742284_1460 (size=17181) 2024-12-05T07:42:11,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384591795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384591799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:11,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384591800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:11,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384591802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:11,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384591906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384591906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:11,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384591916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:11,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384591916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384592116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384592116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384592136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384592136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/58b08705b3094595aaf3245943b1226c 2024-12-05T07:42:12,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/cf0abe7b93c14947a5b61a6ca96cf4f6 is 50, key is test_row_0/B:col10/1733384531764/Put/seqid=0 2024-12-05T07:42:12,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742285_1461 (size=12301) 2024-12-05T07:42:12,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/cf0abe7b93c14947a5b61a6ca96cf4f6 2024-12-05T07:42:12,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/a95d04f611954738b4b3b586dd55ae41 is 50, key is test_row_0/C:col10/1733384531764/Put/seqid=0 2024-12-05T07:42:12,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742286_1462 (size=12301) 2024-12-05T07:42:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T07:42:12,276 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:12,276 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-05T07:42:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T07:42:12,277 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:12,278 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:12,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T07:42:12,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384592426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384592426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T07:42:12,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:12,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:12,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:12,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:12,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:12,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384592446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384592446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T07:42:12,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:12,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:12,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:12,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:12,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:12,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/a95d04f611954738b4b3b586dd55ae41 2024-12-05T07:42:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T07:42:12,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/58b08705b3094595aaf3245943b1226c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/58b08705b3094595aaf3245943b1226c 2024-12-05T07:42:12,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/58b08705b3094595aaf3245943b1226c, entries=250, sequenceid=286, filesize=16.8 K 2024-12-05T07:42:12,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/cf0abe7b93c14947a5b61a6ca96cf4f6 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cf0abe7b93c14947a5b61a6ca96cf4f6 2024-12-05T07:42:12,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cf0abe7b93c14947a5b61a6ca96cf4f6, entries=150, sequenceid=286, filesize=12.0 K 2024-12-05T07:42:12,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/a95d04f611954738b4b3b586dd55ae41 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a95d04f611954738b4b3b586dd55ae41 2024-12-05T07:42:12,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a95d04f611954738b4b3b586dd55ae41, entries=150, sequenceid=286, filesize=12.0 K 2024-12-05T07:42:12,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 45734b659127c96dcd616039a555b19f in 866ms, sequenceid=286, compaction requested=true 2024-12-05T07:42:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:12,632 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:42:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:12,632 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:42:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:12,634 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56720 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:42:12,634 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/A is initiating minor compaction (all files) 2024-12-05T07:42:12,634 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/A in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:12,634 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/713f45a726e747ea9675141d7e0b9e89, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/81352c29392c4dbdaa4de8591669909d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/db4be35ed30a405aa00b7abfb400e908, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/58b08705b3094595aaf3245943b1226c] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=55.4 K 2024-12-05T07:42:12,634 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 713f45a726e747ea9675141d7e0b9e89, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733384529107 2024-12-05T07:42:12,634 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:42:12,634 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/B is initiating minor compaction (all files) 2024-12-05T07:42:12,634 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/B in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:12,634 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8f6776ae5fb4490f971883bc9750e054, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2a7e5538f8d04f4b9b9c8827809f51c4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/5f52eaf121ba4bd8b027d8b49c2375a8, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cf0abe7b93c14947a5b61a6ca96cf4f6] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=48.3 K 2024-12-05T07:42:12,635 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81352c29392c4dbdaa4de8591669909d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733384529500 2024-12-05T07:42:12,635 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f6776ae5fb4490f971883bc9750e054, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733384529107 2024-12-05T07:42:12,635 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a7e5538f8d04f4b9b9c8827809f51c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733384529500 2024-12-05T07:42:12,636 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting db4be35ed30a405aa00b7abfb400e908, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733384529582 2024-12-05T07:42:12,636 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f52eaf121ba4bd8b027d8b49c2375a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733384529582 2024-12-05T07:42:12,637 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting cf0abe7b93c14947a5b61a6ca96cf4f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384531761 2024-12-05T07:42:12,637 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58b08705b3094595aaf3245943b1226c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384531760 2024-12-05T07:42:12,649 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#B#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:12,649 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/8297ee25ee0e43f7a823a4ac1cbc8e96 is 50, key is test_row_0/B:col10/1733384531764/Put/seqid=0 2024-12-05T07:42:12,661 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:12,661 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/5b23cb0f0d374092ac2120cdbd696480 is 50, key is test_row_0/A:col10/1733384531764/Put/seqid=0 2024-12-05T07:42:12,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742288_1464 (size=12983) 2024-12-05T07:42:12,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742287_1463 (size=12983) 2024-12-05T07:42:12,688 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/5b23cb0f0d374092ac2120cdbd696480 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5b23cb0f0d374092ac2120cdbd696480 2024-12-05T07:42:12,689 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/8297ee25ee0e43f7a823a4ac1cbc8e96 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8297ee25ee0e43f7a823a4ac1cbc8e96 2024-12-05T07:42:12,697 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45734b659127c96dcd616039a555b19f/A of 45734b659127c96dcd616039a555b19f into 5b23cb0f0d374092ac2120cdbd696480(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:12,697 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:12,697 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/A, priority=12, startTime=1733384532632; duration=0sec 2024-12-05T07:42:12,697 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:12,697 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:A 2024-12-05T07:42:12,697 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:42:12,701 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:42:12,701 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/C is initiating minor compaction (all files) 2024-12-05T07:42:12,701 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/C in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:12,701 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45734b659127c96dcd616039a555b19f/B of 45734b659127c96dcd616039a555b19f into 8297ee25ee0e43f7a823a4ac1cbc8e96(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:12,701 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:12,701 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/46c9c5567b9f441d819e5895ef478d5e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7f6ad06b766f46578b3df12252293a5d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6bf37804ca2541dc931b3d0c2af3c10e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a95d04f611954738b4b3b586dd55ae41] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=48.3 K 2024-12-05T07:42:12,701 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/B, priority=12, startTime=1733384532632; duration=0sec 2024-12-05T07:42:12,701 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:12,701 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:B 2024-12-05T07:42:12,701 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46c9c5567b9f441d819e5895ef478d5e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733384529107 2024-12-05T07:42:12,702 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f6ad06b766f46578b3df12252293a5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733384529500 2024-12-05T07:42:12,702 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bf37804ca2541dc931b3d0c2af3c10e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733384529582 2024-12-05T07:42:12,702 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a95d04f611954738b4b3b586dd55ae41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384531761 2024-12-05T07:42:12,707 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#C#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:12,708 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/823d232e27d740be9f0aa5919e4a3439 is 50, key is test_row_0/C:col10/1733384531764/Put/seqid=0 2024-12-05T07:42:12,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742289_1465 (size=12983) 2024-12-05T07:42:12,719 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/823d232e27d740be9f0aa5919e4a3439 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/823d232e27d740be9f0aa5919e4a3439 2024-12-05T07:42:12,725 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 45734b659127c96dcd616039a555b19f/C of 45734b659127c96dcd616039a555b19f into 823d232e27d740be9f0aa5919e4a3439(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:12,725 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:12,725 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/C, priority=12, startTime=1733384532632; duration=0sec 2024-12-05T07:42:12,725 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:12,725 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:C 2024-12-05T07:42:12,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T07:42:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:12,736 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T07:42:12,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/5e24c61524a34586bd1f7d163e9b63df is 50, key is test_row_0/A:col10/1733384531801/Put/seqid=0 2024-12-05T07:42:12,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742290_1466 (size=12301) 2024-12-05T07:42:12,772 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/5e24c61524a34586bd1f7d163e9b63df 2024-12-05T07:42:12,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/29fdb2d70dc046d2b95717eb3bebf290 is 50, key is test_row_0/B:col10/1733384531801/Put/seqid=0 2024-12-05T07:42:12,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742291_1467 (size=12301) 2024-12-05T07:42:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T07:42:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:12,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384592959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384592961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384592962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384592963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384593076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384593076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384593076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384593076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,185 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/29fdb2d70dc046d2b95717eb3bebf290 2024-12-05T07:42:13,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/b88cbca5f824453d9b8dca96276418cb is 50, key is test_row_0/C:col10/1733384531801/Put/seqid=0 2024-12-05T07:42:13,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742292_1468 (size=12301) 2024-12-05T07:42:13,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384593286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384593286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384593286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384593286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T07:42:13,594 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/b88cbca5f824453d9b8dca96276418cb 2024-12-05T07:42:13,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/5e24c61524a34586bd1f7d163e9b63df as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5e24c61524a34586bd1f7d163e9b63df 2024-12-05T07:42:13,599 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5e24c61524a34586bd1f7d163e9b63df, entries=150, sequenceid=313, filesize=12.0 K 2024-12-05T07:42:13,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/29fdb2d70dc046d2b95717eb3bebf290 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/29fdb2d70dc046d2b95717eb3bebf290 2024-12-05T07:42:13,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384593595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384593596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384593596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384593596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:13,607 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/29fdb2d70dc046d2b95717eb3bebf290, entries=150, sequenceid=313, filesize=12.0 K 2024-12-05T07:42:13,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/b88cbca5f824453d9b8dca96276418cb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/b88cbca5f824453d9b8dca96276418cb 2024-12-05T07:42:13,610 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/b88cbca5f824453d9b8dca96276418cb, entries=150, sequenceid=313, filesize=12.0 K 2024-12-05T07:42:13,611 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 45734b659127c96dcd616039a555b19f in 875ms, sequenceid=313, compaction requested=false 2024-12-05T07:42:13,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:13,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:13,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-05T07:42:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-05T07:42:13,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-05T07:42:13,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3340 sec 2024-12-05T07:42:13,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3380 sec 2024-12-05T07:42:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:14,119 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:14,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-12-05T07:42:14,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/850855b7d4f24155b27ea9d55b77321c is 50, key is test_row_1/A:col10/1733384532961/Put/seqid=0 2024-12-05T07:42:14,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742293_1469 (size=9857) 2024-12-05T07:42:14,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/850855b7d4f24155b27ea9d55b77321c 2024-12-05T07:42:14,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/802aa99e5b23437bb03e0c242d98f869 is 50, key is test_row_1/B:col10/1733384532961/Put/seqid=0 2024-12-05T07:42:14,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742294_1470 (size=9857) 2024-12-05T07:42:14,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384594195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384594196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384594197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384594197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384594305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384594316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384594316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384594316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T07:42:14,426 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:14,427 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-05T07:42:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T07:42:14,428 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:14,429 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:14,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:14,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384594516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384594526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384594526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384594526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T07:42:14,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/802aa99e5b23437bb03e0c242d98f869 2024-12-05T07:42:14,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/8d2786ef910041139f6dbe6cc3cb6ffa is 50, key is test_row_1/C:col10/1733384532961/Put/seqid=0 2024-12-05T07:42:14,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742295_1471 (size=9857) 2024-12-05T07:42:14,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T07:42:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:14,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:14,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:14,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T07:42:14,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:14,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:14,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T07:42:14,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384594825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384594836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384594836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384594836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:14,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T07:42:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:14,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:14,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:14,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/8d2786ef910041139f6dbe6cc3cb6ffa 2024-12-05T07:42:14,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/850855b7d4f24155b27ea9d55b77321c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/850855b7d4f24155b27ea9d55b77321c 2024-12-05T07:42:14,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/850855b7d4f24155b27ea9d55b77321c, entries=100, sequenceid=326, filesize=9.6 K 2024-12-05T07:42:14,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/802aa99e5b23437bb03e0c242d98f869 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/802aa99e5b23437bb03e0c242d98f869 2024-12-05T07:42:14,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/802aa99e5b23437bb03e0c242d98f869, entries=100, sequenceid=326, filesize=9.6 K 2024-12-05T07:42:14,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/8d2786ef910041139f6dbe6cc3cb6ffa as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/8d2786ef910041139f6dbe6cc3cb6ffa 2024-12-05T07:42:14,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/8d2786ef910041139f6dbe6cc3cb6ffa, entries=100, sequenceid=326, filesize=9.6 K 2024-12-05T07:42:14,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 45734b659127c96dcd616039a555b19f in 854ms, sequenceid=326, compaction requested=true 2024-12-05T07:42:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:14,974 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:14,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:14,974 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:14,974 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:14,974 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:14,974 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/A is initiating minor compaction (all files) 2024-12-05T07:42:14,974 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/B is initiating minor compaction (all files) 2024-12-05T07:42:14,974 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/A in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:14,974 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/B in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:14,974 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5b23cb0f0d374092ac2120cdbd696480, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5e24c61524a34586bd1f7d163e9b63df, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/850855b7d4f24155b27ea9d55b77321c] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=34.3 K 2024-12-05T07:42:14,975 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8297ee25ee0e43f7a823a4ac1cbc8e96, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/29fdb2d70dc046d2b95717eb3bebf290, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/802aa99e5b23437bb03e0c242d98f869] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=34.3 K 2024-12-05T07:42:14,975 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b23cb0f0d374092ac2120cdbd696480, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384531761 2024-12-05T07:42:14,975 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8297ee25ee0e43f7a823a4ac1cbc8e96, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384531761 2024-12-05T07:42:14,975 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e24c61524a34586bd1f7d163e9b63df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733384531792 2024-12-05T07:42:14,975 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 29fdb2d70dc046d2b95717eb3bebf290, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733384531792 2024-12-05T07:42:14,975 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 850855b7d4f24155b27ea9d55b77321c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733384532961 2024-12-05T07:42:14,975 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 802aa99e5b23437bb03e0c242d98f869, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733384532961 2024-12-05T07:42:14,981 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#A#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:14,981 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:14,981 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/9b32700abcdb4d46ae557edd900b327e is 50, key is test_row_0/A:col10/1733384531801/Put/seqid=0 2024-12-05T07:42:14,981 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/9082f00cb33d41448984999ddb99c45d is 50, key is test_row_0/B:col10/1733384531801/Put/seqid=0 2024-12-05T07:42:14,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742297_1473 (size=13085) 2024-12-05T07:42:14,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742296_1472 (size=13085) 2024-12-05T07:42:15,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T07:42:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:15,040 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T07:42:15,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/a19775b0430c477a8313ac44db3b982f is 50, key is test_row_0/A:col10/1733384534196/Put/seqid=0 2024-12-05T07:42:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T07:42:15,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742298_1474 (size=12301) 2024-12-05T07:42:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:15,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:15,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384595367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384595367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384595371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384595375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,397 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/9082f00cb33d41448984999ddb99c45d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/9082f00cb33d41448984999ddb99c45d 2024-12-05T07:42:15,399 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/9b32700abcdb4d46ae557edd900b327e as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/9b32700abcdb4d46ae557edd900b327e 2024-12-05T07:42:15,401 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/B of 45734b659127c96dcd616039a555b19f into 9082f00cb33d41448984999ddb99c45d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:15,401 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:15,401 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/B, priority=13, startTime=1733384534974; duration=0sec 2024-12-05T07:42:15,401 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:15,401 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:B 2024-12-05T07:42:15,401 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:15,402 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:15,402 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/C is initiating minor compaction (all files) 2024-12-05T07:42:15,402 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/C in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:15,402 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/823d232e27d740be9f0aa5919e4a3439, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/b88cbca5f824453d9b8dca96276418cb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/8d2786ef910041139f6dbe6cc3cb6ffa] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=34.3 K 2024-12-05T07:42:15,403 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 823d232e27d740be9f0aa5919e4a3439, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733384531761 2024-12-05T07:42:15,403 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/A of 45734b659127c96dcd616039a555b19f into 9b32700abcdb4d46ae557edd900b327e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:15,403 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:15,403 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/A, priority=13, startTime=1733384534974; duration=0sec 2024-12-05T07:42:15,403 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:15,403 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:A 2024-12-05T07:42:15,405 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting b88cbca5f824453d9b8dca96276418cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1733384531792 2024-12-05T07:42:15,405 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2786ef910041139f6dbe6cc3cb6ffa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733384532961 2024-12-05T07:42:15,411 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:15,411 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/9ee3e27eb85a432f802718e9e1b79b56 is 50, key is test_row_0/C:col10/1733384531801/Put/seqid=0 2024-12-05T07:42:15,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742299_1475 (size=13085) 2024-12-05T07:42:15,461 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/a19775b0430c477a8313ac44db3b982f 2024-12-05T07:42:15,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/d9080f62e4c54f35af9b06165b5c22ab is 50, key is test_row_0/B:col10/1733384534196/Put/seqid=0 2024-12-05T07:42:15,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742300_1476 (size=12301) 2024-12-05T07:42:15,476 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/d9080f62e4c54f35af9b06165b5c22ab 2024-12-05T07:42:15,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/a25568fa77e44acc9cfb39ef65470785 is 50, key is test_row_0/C:col10/1733384534196/Put/seqid=0 2024-12-05T07:42:15,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384595486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384595486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384595487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384595487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742301_1477 (size=12301) 2024-12-05T07:42:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T07:42:15,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384595697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384595697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384595698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384595697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:15,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:15,821 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/9ee3e27eb85a432f802718e9e1b79b56 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/9ee3e27eb85a432f802718e9e1b79b56 2024-12-05T07:42:15,826 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/C of 45734b659127c96dcd616039a555b19f into 9ee3e27eb85a432f802718e9e1b79b56(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:15,826 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:15,826 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/C, priority=13, startTime=1733384534974; duration=0sec 2024-12-05T07:42:15,826 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:15,826 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:C 2024-12-05T07:42:15,893 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/a25568fa77e44acc9cfb39ef65470785 2024-12-05T07:42:15,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/a19775b0430c477a8313ac44db3b982f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a19775b0430c477a8313ac44db3b982f 2024-12-05T07:42:15,906 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a19775b0430c477a8313ac44db3b982f, entries=150, sequenceid=349, filesize=12.0 K 2024-12-05T07:42:15,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/d9080f62e4c54f35af9b06165b5c22ab as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/d9080f62e4c54f35af9b06165b5c22ab 2024-12-05T07:42:15,910 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/d9080f62e4c54f35af9b06165b5c22ab, entries=150, sequenceid=349, filesize=12.0 K 2024-12-05T07:42:15,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/a25568fa77e44acc9cfb39ef65470785 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a25568fa77e44acc9cfb39ef65470785 2024-12-05T07:42:15,912 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a25568fa77e44acc9cfb39ef65470785, entries=150, sequenceid=349, filesize=12.0 K 2024-12-05T07:42:15,913 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 45734b659127c96dcd616039a555b19f in 873ms, sequenceid=349, compaction requested=false 2024-12-05T07:42:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:15,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-05T07:42:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-05T07:42:15,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-05T07:42:15,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4850 sec 2024-12-05T07:42:15,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4880 sec 2024-12-05T07:42:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:16,019 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:16,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T07:42:16,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/7b8a250dc7a546ffa9b09b9368ab3c45 is 50, key is test_row_0/A:col10/1733384536019/Put/seqid=0 2024-12-05T07:42:16,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742302_1478 (size=17181) 2024-12-05T07:42:16,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384596052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384596054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384596055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384596062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384596166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384596166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384596166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384596166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384596376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384596376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384596376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384596376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/7b8a250dc7a546ffa9b09b9368ab3c45 2024-12-05T07:42:16,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/633d0f3cf35446fe967a906c266e4cda is 50, key is test_row_0/B:col10/1733384536019/Put/seqid=0 2024-12-05T07:42:16,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742303_1479 (size=12301) 2024-12-05T07:42:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T07:42:16,576 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:16,576 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-05T07:42:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T07:42:16,580 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:16,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:16,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T07:42:16,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384596696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384596696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384596696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384596696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T07:42:16,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:16,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:16,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:16,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:16,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:16,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:16,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33002 deadline: 1733384596826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:16,830 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in 45734b659127c96dcd616039a555b19f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18320 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:16,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/633d0f3cf35446fe967a906c266e4cda 2024-12-05T07:42:16,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/f3d6dd4eb2b64a24884e7bd029630eae is 50, key is test_row_0/C:col10/1733384536019/Put/seqid=0 2024-12-05T07:42:16,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742304_1480 (size=12301) 2024-12-05T07:42:16,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T07:42:16,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:16,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:16,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:16,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:16,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:16,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T07:42:17,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T07:42:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:17,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:17,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:17,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T07:42:17,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:17,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:17,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:17,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:17,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:17,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32952 deadline: 1733384597206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:17,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32938 deadline: 1733384597206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:17,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32986 deadline: 1733384597206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:17,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T07:42:17,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32918 deadline: 1733384597206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:17,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:17,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:17,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., hostname=fc6cd38557f3,36693,1733384394401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T07:42:17,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/f3d6dd4eb2b64a24884e7bd029630eae 2024-12-05T07:42:17,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/7b8a250dc7a546ffa9b09b9368ab3c45 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/7b8a250dc7a546ffa9b09b9368ab3c45 2024-12-05T07:42:17,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/7b8a250dc7a546ffa9b09b9368ab3c45, entries=250, sequenceid=367, filesize=16.8 K 2024-12-05T07:42:17,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/633d0f3cf35446fe967a906c266e4cda as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/633d0f3cf35446fe967a906c266e4cda 2024-12-05T07:42:17,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/633d0f3cf35446fe967a906c266e4cda, entries=150, sequenceid=367, filesize=12.0 K 2024-12-05T07:42:17,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/f3d6dd4eb2b64a24884e7bd029630eae as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f3d6dd4eb2b64a24884e7bd029630eae 2024-12-05T07:42:17,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f3d6dd4eb2b64a24884e7bd029630eae, entries=150, sequenceid=367, filesize=12.0 K 2024-12-05T07:42:17,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 45734b659127c96dcd616039a555b19f in 1247ms, sequenceid=367, compaction requested=true 2024-12-05T07:42:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:17,266 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:17,266 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45734b659127c96dcd616039a555b19f:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:17,266 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:17,267 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:17,267 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:17,267 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/A is initiating minor compaction (all files) 2024-12-05T07:42:17,267 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/B is initiating minor compaction (all files) 2024-12-05T07:42:17,267 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/A in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:17,267 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/B in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:17,267 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/9b32700abcdb4d46ae557edd900b327e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a19775b0430c477a8313ac44db3b982f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/7b8a250dc7a546ffa9b09b9368ab3c45] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=41.6 K 2024-12-05T07:42:17,267 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/9082f00cb33d41448984999ddb99c45d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/d9080f62e4c54f35af9b06165b5c22ab, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/633d0f3cf35446fe967a906c266e4cda] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=36.8 K 2024-12-05T07:42:17,267 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b32700abcdb4d46ae557edd900b327e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733384531801 2024-12-05T07:42:17,267 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 9082f00cb33d41448984999ddb99c45d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733384531801 2024-12-05T07:42:17,268 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a19775b0430c477a8313ac44db3b982f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733384534193 2024-12-05T07:42:17,269 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting d9080f62e4c54f35af9b06165b5c22ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733384534193 2024-12-05T07:42:17,269 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b8a250dc7a546ffa9b09b9368ab3c45, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733384535365 2024-12-05T07:42:17,269 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 633d0f3cf35446fe967a906c266e4cda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733384535365 2024-12-05T07:42:17,288 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#A#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:17,288 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/f45f6f18c7d94e63a60d02b5a9048e5f is 50, key is test_row_0/A:col10/1733384536019/Put/seqid=0 2024-12-05T07:42:17,292 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#B#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:17,293 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/c673e54e63b34aec822bcd5a923ad8bb is 50, key is test_row_0/B:col10/1733384536019/Put/seqid=0 2024-12-05T07:42:17,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742305_1481 (size=13187) 2024-12-05T07:42:17,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742306_1482 (size=13187) 2024-12-05T07:42:17,320 INFO [Thread-1837 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1837. 2024-12-05T07:42:17,320 DEBUG [Thread-1837 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:17,320 DEBUG [Thread-1837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:17,320 DEBUG [Thread-1837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:17,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:17,322 INFO [Thread-1843 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1843. 2024-12-05T07:42:17,322 DEBUG [Thread-1843 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:17,322 DEBUG [Thread-1843 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:17,322 DEBUG [Thread-1843 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:17,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:17,324 INFO [Thread-1841 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1841. 2024-12-05T07:42:17,324 DEBUG [Thread-1841 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:17,324 DEBUG [Thread-1841 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:17,324 DEBUG [Thread-1841 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:17,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:17,327 INFO [Thread-1839 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1839. 2024-12-05T07:42:17,327 DEBUG [Thread-1839 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:17,327 DEBUG [Thread-1839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:17,327 DEBUG [Thread-1839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:17,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:17,328 INFO [Thread-1835 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1835. 2024-12-05T07:42:17,328 DEBUG [Thread-1835 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:17,328 DEBUG [Thread-1835 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:17,328 DEBUG [Thread-1835 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:17,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:17,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T07:42:17,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:17,344 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T07:42:17,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/fc9351ded1254d88a0f732f890f692d2 is 50, key is test_row_0/A:col10/1733384536061/Put/seqid=0 2024-12-05T07:42:17,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742307_1483 (size=12301) 2024-12-05T07:42:17,702 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/f45f6f18c7d94e63a60d02b5a9048e5f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/f45f6f18c7d94e63a60d02b5a9048e5f 2024-12-05T07:42:17,705 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/c673e54e63b34aec822bcd5a923ad8bb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/c673e54e63b34aec822bcd5a923ad8bb 2024-12-05T07:42:17,710 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/A of 45734b659127c96dcd616039a555b19f into f45f6f18c7d94e63a60d02b5a9048e5f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:17,710 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:17,710 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/A, priority=13, startTime=1733384537266; duration=0sec 2024-12-05T07:42:17,710 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:17,710 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:A 2024-12-05T07:42:17,710 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:17,711 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:17,711 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): 45734b659127c96dcd616039a555b19f/C is initiating minor compaction (all files) 2024-12-05T07:42:17,711 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45734b659127c96dcd616039a555b19f/C in TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:17,711 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/B of 45734b659127c96dcd616039a555b19f into c673e54e63b34aec822bcd5a923ad8bb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:17,711 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:17,711 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/9ee3e27eb85a432f802718e9e1b79b56, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a25568fa77e44acc9cfb39ef65470785, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f3d6dd4eb2b64a24884e7bd029630eae] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp, totalSize=36.8 K 2024-12-05T07:42:17,711 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/B, priority=13, startTime=1733384537266; duration=0sec 2024-12-05T07:42:17,711 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:17,711 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:B 2024-12-05T07:42:17,711 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ee3e27eb85a432f802718e9e1b79b56, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733384531801 2024-12-05T07:42:17,711 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a25568fa77e44acc9cfb39ef65470785, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733384534193 2024-12-05T07:42:17,712 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3d6dd4eb2b64a24884e7bd029630eae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733384535365 2024-12-05T07:42:17,716 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45734b659127c96dcd616039a555b19f#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:17,716 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/1c826a0e89d044c082da981558f48e82 is 50, key is test_row_0/C:col10/1733384536019/Put/seqid=0 2024-12-05T07:42:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T07:42:17,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742308_1484 (size=13187) 2024-12-05T07:42:17,752 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/fc9351ded1254d88a0f732f890f692d2 2024-12-05T07:42:17,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/532227c255904d50beebbd6e3a31c319 is 50, key is test_row_0/B:col10/1733384536061/Put/seqid=0 2024-12-05T07:42:17,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742309_1485 (size=12301) 2024-12-05T07:42:18,129 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/1c826a0e89d044c082da981558f48e82 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/1c826a0e89d044c082da981558f48e82 2024-12-05T07:42:18,134 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45734b659127c96dcd616039a555b19f/C of 45734b659127c96dcd616039a555b19f into 1c826a0e89d044c082da981558f48e82(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:18,134 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:18,134 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., storeName=45734b659127c96dcd616039a555b19f/C, priority=13, startTime=1733384537266; duration=0sec 2024-12-05T07:42:18,134 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:18,134 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45734b659127c96dcd616039a555b19f:C 2024-12-05T07:42:18,160 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/532227c255904d50beebbd6e3a31c319 2024-12-05T07:42:18,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/2e1120bdddbf484493d77f1274fe7c94 is 50, key is test_row_0/C:col10/1733384536061/Put/seqid=0 2024-12-05T07:42:18,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742310_1486 (size=12301) 2024-12-05T07:42:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:18,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. as already flushing 2024-12-05T07:42:18,220 INFO [Thread-1830 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1830. 2024-12-05T07:42:18,220 DEBUG [Thread-1830 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:18,220 INFO [Thread-1826 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1826. 2024-12-05T07:42:18,220 DEBUG [Thread-1830 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:18,221 DEBUG [Thread-1826 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:18,221 DEBUG [Thread-1826 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:18,221 DEBUG [Thread-1830 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:18,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:18,221 DEBUG [Thread-1826 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:18,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:18,228 INFO [Thread-1824 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1824. 2024-12-05T07:42:18,228 INFO [Thread-1828 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1828. 2024-12-05T07:42:18,228 DEBUG [Thread-1824 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:18,228 DEBUG [Thread-1828 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:18,228 DEBUG [Thread-1824 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:18,228 DEBUG [Thread-1828 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:18,228 DEBUG [Thread-1824 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:18,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:18,228 DEBUG [Thread-1828 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:18,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:18,569 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/2e1120bdddbf484493d77f1274fe7c94 2024-12-05T07:42:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/fc9351ded1254d88a0f732f890f692d2 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/fc9351ded1254d88a0f732f890f692d2 2024-12-05T07:42:18,583 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/fc9351ded1254d88a0f732f890f692d2, entries=150, sequenceid=388, filesize=12.0 K 2024-12-05T07:42:18,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/532227c255904d50beebbd6e3a31c319 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/532227c255904d50beebbd6e3a31c319 2024-12-05T07:42:18,589 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/532227c255904d50beebbd6e3a31c319, entries=150, sequenceid=388, filesize=12.0 K 2024-12-05T07:42:18,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/2e1120bdddbf484493d77f1274fe7c94 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2e1120bdddbf484493d77f1274fe7c94 2024-12-05T07:42:18,594 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2e1120bdddbf484493d77f1274fe7c94, entries=150, sequenceid=388, filesize=12.0 K 2024-12-05T07:42:18,595 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=26.84 KB/27480 for 45734b659127c96dcd616039a555b19f in 1251ms, sequenceid=388, compaction requested=false 2024-12-05T07:42:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 45734b659127c96dcd616039a555b19f: 2024-12-05T07:42:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:18,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-05T07:42:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-05T07:42:18,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-05T07:42:18,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0140 sec 2024-12-05T07:42:18,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.0210 sec 2024-12-05T07:42:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T07:42:18,727 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:22,575 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T07:42:26,858 INFO [Thread-1832 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1832. 2024-12-05T07:42:26,858 DEBUG [Thread-1832 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:26,858 DEBUG [Thread-1832 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:26,858 DEBUG [Thread-1832 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:26,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 149 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2093 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6274 rows 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2078 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6234 rows 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2085 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6255 rows 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2092 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6276 rows 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2080 2024-12-05T07:42:26,858 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6240 rows 2024-12-05T07:42:26,859 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T07:42:26,859 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T07:42:26,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:26,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:26,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:26,860 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T07:42:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T07:42:26,864 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384546864"}]},"ts":"1733384546864"} 2024-12-05T07:42:26,866 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T07:42:26,866 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T07:42:26,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T07:42:26,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45734b659127c96dcd616039a555b19f, UNASSIGN}] 2024-12-05T07:42:26,870 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45734b659127c96dcd616039a555b19f, UNASSIGN 2024-12-05T07:42:26,870 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=45734b659127c96dcd616039a555b19f, regionState=CLOSING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:26,871 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33643 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=fc6cd38557f3,36693,1733384394401, table=TestAcidGuarantees, region=45734b659127c96dcd616039a555b19f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T07:42:26,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45734b659127c96dcd616039a555b19f, UNASSIGN because future has completed 2024-12-05T07:42:26,872 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T07:42:26,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:42:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T07:42:27,025 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:27,025 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T07:42:27,025 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing 45734b659127c96dcd616039a555b19f, disabling compactions & flushes 2024-12-05T07:42:27,025 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:27,025 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:27,025 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. after waiting 0 ms 2024-12-05T07:42:27,025 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:27,025 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(2902): Flushing 45734b659127c96dcd616039a555b19f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-05T07:42:27,030 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/5ecff829c2244f678126519f3cddd6bd is 50, key is test_row_1/A:col10/1733384538226/Put/seqid=0 2024-12-05T07:42:27,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742311_1487 (size=9857) 2024-12-05T07:42:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T07:42:27,436 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/5ecff829c2244f678126519f3cddd6bd 2024-12-05T07:42:27,451 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/64b3131db6be4a6ea9d6608dbc3c611d is 50, key is test_row_1/B:col10/1733384538226/Put/seqid=0 2024-12-05T07:42:27,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742312_1488 (size=9857) 2024-12-05T07:42:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T07:42:27,857 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/64b3131db6be4a6ea9d6608dbc3c611d 2024-12-05T07:42:27,872 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/598eba368a1c4a47a5194d8fbacbd683 is 50, key is test_row_1/C:col10/1733384538226/Put/seqid=0 2024-12-05T07:42:27,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742313_1489 (size=9857) 2024-12-05T07:42:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T07:42:28,276 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/598eba368a1c4a47a5194d8fbacbd683 2024-12-05T07:42:28,330 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/A/5ecff829c2244f678126519f3cddd6bd as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5ecff829c2244f678126519f3cddd6bd 2024-12-05T07:42:28,333 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5ecff829c2244f678126519f3cddd6bd, entries=100, sequenceid=399, filesize=9.6 K 2024-12-05T07:42:28,334 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/B/64b3131db6be4a6ea9d6608dbc3c611d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/64b3131db6be4a6ea9d6608dbc3c611d 2024-12-05T07:42:28,336 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/64b3131db6be4a6ea9d6608dbc3c611d, entries=100, sequenceid=399, filesize=9.6 K 2024-12-05T07:42:28,337 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/.tmp/C/598eba368a1c4a47a5194d8fbacbd683 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/598eba368a1c4a47a5194d8fbacbd683 2024-12-05T07:42:28,339 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/598eba368a1c4a47a5194d8fbacbd683, entries=100, sequenceid=399, filesize=9.6 K 2024-12-05T07:42:28,340 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 45734b659127c96dcd616039a555b19f in 1315ms, sequenceid=399, compaction requested=true 2024-12-05T07:42:28,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a98f5922e0384cac9045b69b7d1a6bfb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/10439ee63a9948d2abcc0a95fc1d6364, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/18fbd2cf50fb4090bfaaf0dc0a6356a5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5c14fdac638746ccb547ca54125145d1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/687e677a709a4c76a8da017701b2012c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/21afda56e7b849e2b12af1a87977af21, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/2fb63c33b0734109bbdebb96210da1c3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/b2b98a63360d47c99a3b41bae281cfbb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d6f4bed34d134158bc896205d0a4c4c6, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/bc0b4949075e42e4b7e765ea86835c98, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/329bb74e567446779474f2bb6367365b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/4c406b5e58f94c27bfe1f93f1b57fa71, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d4c74279041948e984bc543fd0fc52bc, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/476663e650e44fef9560fc276b045ef2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/f25176863770402c906bddc7f3170871, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/8ab369edb81a4fbba816bb1fa3499e95, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/713f45a726e747ea9675141d7e0b9e89, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/81352c29392c4dbdaa4de8591669909d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/db4be35ed30a405aa00b7abfb400e908, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/58b08705b3094595aaf3245943b1226c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5b23cb0f0d374092ac2120cdbd696480, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5e24c61524a34586bd1f7d163e9b63df, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/9b32700abcdb4d46ae557edd900b327e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/850855b7d4f24155b27ea9d55b77321c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a19775b0430c477a8313ac44db3b982f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/7b8a250dc7a546ffa9b09b9368ab3c45] to archive 2024-12-05T07:42:28,341 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:42:28,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a98f5922e0384cac9045b69b7d1a6bfb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a98f5922e0384cac9045b69b7d1a6bfb 2024-12-05T07:42:28,343 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/10439ee63a9948d2abcc0a95fc1d6364 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/10439ee63a9948d2abcc0a95fc1d6364 2024-12-05T07:42:28,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/18fbd2cf50fb4090bfaaf0dc0a6356a5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/18fbd2cf50fb4090bfaaf0dc0a6356a5 2024-12-05T07:42:28,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5c14fdac638746ccb547ca54125145d1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5c14fdac638746ccb547ca54125145d1 2024-12-05T07:42:28,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/687e677a709a4c76a8da017701b2012c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/687e677a709a4c76a8da017701b2012c 2024-12-05T07:42:28,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/21afda56e7b849e2b12af1a87977af21 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/21afda56e7b849e2b12af1a87977af21 2024-12-05T07:42:28,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/2fb63c33b0734109bbdebb96210da1c3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/2fb63c33b0734109bbdebb96210da1c3 2024-12-05T07:42:28,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/b2b98a63360d47c99a3b41bae281cfbb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/b2b98a63360d47c99a3b41bae281cfbb 2024-12-05T07:42:28,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d6f4bed34d134158bc896205d0a4c4c6 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d6f4bed34d134158bc896205d0a4c4c6 2024-12-05T07:42:28,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/bc0b4949075e42e4b7e765ea86835c98 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/bc0b4949075e42e4b7e765ea86835c98 2024-12-05T07:42:28,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/329bb74e567446779474f2bb6367365b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/329bb74e567446779474f2bb6367365b 2024-12-05T07:42:28,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/4c406b5e58f94c27bfe1f93f1b57fa71 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/4c406b5e58f94c27bfe1f93f1b57fa71 2024-12-05T07:42:28,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d4c74279041948e984bc543fd0fc52bc to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/d4c74279041948e984bc543fd0fc52bc 2024-12-05T07:42:28,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/476663e650e44fef9560fc276b045ef2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/476663e650e44fef9560fc276b045ef2 2024-12-05T07:42:28,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/f25176863770402c906bddc7f3170871 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/f25176863770402c906bddc7f3170871 2024-12-05T07:42:28,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/8ab369edb81a4fbba816bb1fa3499e95 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/8ab369edb81a4fbba816bb1fa3499e95 2024-12-05T07:42:28,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/713f45a726e747ea9675141d7e0b9e89 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/713f45a726e747ea9675141d7e0b9e89 2024-12-05T07:42:28,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/81352c29392c4dbdaa4de8591669909d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/81352c29392c4dbdaa4de8591669909d 2024-12-05T07:42:28,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/db4be35ed30a405aa00b7abfb400e908 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/db4be35ed30a405aa00b7abfb400e908 2024-12-05T07:42:28,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/58b08705b3094595aaf3245943b1226c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/58b08705b3094595aaf3245943b1226c 2024-12-05T07:42:28,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5b23cb0f0d374092ac2120cdbd696480 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5b23cb0f0d374092ac2120cdbd696480 2024-12-05T07:42:28,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5e24c61524a34586bd1f7d163e9b63df to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5e24c61524a34586bd1f7d163e9b63df 2024-12-05T07:42:28,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/9b32700abcdb4d46ae557edd900b327e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/9b32700abcdb4d46ae557edd900b327e 2024-12-05T07:42:28,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/850855b7d4f24155b27ea9d55b77321c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/850855b7d4f24155b27ea9d55b77321c 2024-12-05T07:42:28,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a19775b0430c477a8313ac44db3b982f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/a19775b0430c477a8313ac44db3b982f 2024-12-05T07:42:28,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/7b8a250dc7a546ffa9b09b9368ab3c45 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/7b8a250dc7a546ffa9b09b9368ab3c45 2024-12-05T07:42:28,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/bc68405e53df4d489b2d3450a7c513d1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7adce838cd984f388ec1816e8e1bb757, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/859e7adbcc184490a22677dd447b308f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7ef69cff6dc24085a7b83657673af1d8, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/767ebaf8a9de4cbfb3ac203df0c21a4d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/78eb43e1762f4225b239b2acbd2aaea4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/48f038d3e9c344088030f207b514233b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/fe1c91359a6d42588e440e231decfbae, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/05c51842745b40c989f65b474a329bca, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/49ead8faa2994789bfe18ad740818ae1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8b907d2198da4e9fbb8dc28df428eb2e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cd21986985d84db986fe742b00ce3f69, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/800441b784364514820cad8f92f9f034, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2b05b360fb5945a69523d9ae792dba6d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/0bfe1ec0801c49229d650b9496c0cbd7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8f6776ae5fb4490f971883bc9750e054, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/c8899b2e7d38453da5419ffdd9ef18c6, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2a7e5538f8d04f4b9b9c8827809f51c4, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/5f52eaf121ba4bd8b027d8b49c2375a8, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8297ee25ee0e43f7a823a4ac1cbc8e96, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cf0abe7b93c14947a5b61a6ca96cf4f6, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/29fdb2d70dc046d2b95717eb3bebf290, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/9082f00cb33d41448984999ddb99c45d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/802aa99e5b23437bb03e0c242d98f869, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/d9080f62e4c54f35af9b06165b5c22ab, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/633d0f3cf35446fe967a906c266e4cda] to archive 2024-12-05T07:42:28,362 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:42:28,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/bc68405e53df4d489b2d3450a7c513d1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/bc68405e53df4d489b2d3450a7c513d1 2024-12-05T07:42:28,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7adce838cd984f388ec1816e8e1bb757 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7adce838cd984f388ec1816e8e1bb757 2024-12-05T07:42:28,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/859e7adbcc184490a22677dd447b308f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/859e7adbcc184490a22677dd447b308f 2024-12-05T07:42:28,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7ef69cff6dc24085a7b83657673af1d8 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/7ef69cff6dc24085a7b83657673af1d8 2024-12-05T07:42:28,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/767ebaf8a9de4cbfb3ac203df0c21a4d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/767ebaf8a9de4cbfb3ac203df0c21a4d 2024-12-05T07:42:28,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/78eb43e1762f4225b239b2acbd2aaea4 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/78eb43e1762f4225b239b2acbd2aaea4 2024-12-05T07:42:28,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/48f038d3e9c344088030f207b514233b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/48f038d3e9c344088030f207b514233b 2024-12-05T07:42:28,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/fe1c91359a6d42588e440e231decfbae to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/fe1c91359a6d42588e440e231decfbae 2024-12-05T07:42:28,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/05c51842745b40c989f65b474a329bca to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/05c51842745b40c989f65b474a329bca 2024-12-05T07:42:28,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/49ead8faa2994789bfe18ad740818ae1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/49ead8faa2994789bfe18ad740818ae1 2024-12-05T07:42:28,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8b907d2198da4e9fbb8dc28df428eb2e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8b907d2198da4e9fbb8dc28df428eb2e 2024-12-05T07:42:28,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cd21986985d84db986fe742b00ce3f69 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cd21986985d84db986fe742b00ce3f69 2024-12-05T07:42:28,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/800441b784364514820cad8f92f9f034 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/800441b784364514820cad8f92f9f034 2024-12-05T07:42:28,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2b05b360fb5945a69523d9ae792dba6d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2b05b360fb5945a69523d9ae792dba6d 2024-12-05T07:42:28,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/0bfe1ec0801c49229d650b9496c0cbd7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/0bfe1ec0801c49229d650b9496c0cbd7 2024-12-05T07:42:28,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8f6776ae5fb4490f971883bc9750e054 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8f6776ae5fb4490f971883bc9750e054 2024-12-05T07:42:28,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/c8899b2e7d38453da5419ffdd9ef18c6 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/c8899b2e7d38453da5419ffdd9ef18c6 2024-12-05T07:42:28,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2a7e5538f8d04f4b9b9c8827809f51c4 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/2a7e5538f8d04f4b9b9c8827809f51c4 2024-12-05T07:42:28,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/5f52eaf121ba4bd8b027d8b49c2375a8 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/5f52eaf121ba4bd8b027d8b49c2375a8 2024-12-05T07:42:28,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8297ee25ee0e43f7a823a4ac1cbc8e96 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/8297ee25ee0e43f7a823a4ac1cbc8e96 2024-12-05T07:42:28,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cf0abe7b93c14947a5b61a6ca96cf4f6 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/cf0abe7b93c14947a5b61a6ca96cf4f6 2024-12-05T07:42:28,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/29fdb2d70dc046d2b95717eb3bebf290 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/29fdb2d70dc046d2b95717eb3bebf290 2024-12-05T07:42:28,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/9082f00cb33d41448984999ddb99c45d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/9082f00cb33d41448984999ddb99c45d 2024-12-05T07:42:28,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/802aa99e5b23437bb03e0c242d98f869 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/802aa99e5b23437bb03e0c242d98f869 2024-12-05T07:42:28,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/d9080f62e4c54f35af9b06165b5c22ab to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/d9080f62e4c54f35af9b06165b5c22ab 2024-12-05T07:42:28,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/633d0f3cf35446fe967a906c266e4cda to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/633d0f3cf35446fe967a906c266e4cda 2024-12-05T07:42:28,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/57cf46f983e243c8b4b05671061ccae7, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/dc5b707e97d348519a9bbd8163059284, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/5b09548714f0468cbc87d77f2572ee62, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/215bd4e650b74701928d556689b1a04e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/0b2cef2b62ee48f28eaefc801d4cd845, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7ebbd3899a0e459c8ebd953295aa3658, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a1e713fe62ac4331a107e0a32b18b3f1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f8a18d9f5f0c44aeb067571c70066666, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/263867a76de440dc8ee0f6d017dbce16, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f753451c958045bcb22b29abd8d8476f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/37e1694d3358453488ae43d26f5c3f7e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2a4a4feb75e94abf9de7e32dc60fd254, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2231745005204b50a528e6572573a7a2, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f93e8facd15340bea9417b55d799e7d1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6a47bd24af4f4ee4b0ccfcf878f93597, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/46c9c5567b9f441d819e5895ef478d5e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a48328e8d7c44e4f8d5db859e16c438a, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7f6ad06b766f46578b3df12252293a5d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6bf37804ca2541dc931b3d0c2af3c10e, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/823d232e27d740be9f0aa5919e4a3439, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a95d04f611954738b4b3b586dd55ae41, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/b88cbca5f824453d9b8dca96276418cb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/9ee3e27eb85a432f802718e9e1b79b56, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/8d2786ef910041139f6dbe6cc3cb6ffa, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a25568fa77e44acc9cfb39ef65470785, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f3d6dd4eb2b64a24884e7bd029630eae] to archive 2024-12-05T07:42:28,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:42:28,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/57cf46f983e243c8b4b05671061ccae7 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/57cf46f983e243c8b4b05671061ccae7 2024-12-05T07:42:28,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/dc5b707e97d348519a9bbd8163059284 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/dc5b707e97d348519a9bbd8163059284 2024-12-05T07:42:28,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/5b09548714f0468cbc87d77f2572ee62 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/5b09548714f0468cbc87d77f2572ee62 2024-12-05T07:42:28,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/215bd4e650b74701928d556689b1a04e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/215bd4e650b74701928d556689b1a04e 2024-12-05T07:42:28,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/0b2cef2b62ee48f28eaefc801d4cd845 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/0b2cef2b62ee48f28eaefc801d4cd845 2024-12-05T07:42:28,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7ebbd3899a0e459c8ebd953295aa3658 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7ebbd3899a0e459c8ebd953295aa3658 2024-12-05T07:42:28,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a1e713fe62ac4331a107e0a32b18b3f1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a1e713fe62ac4331a107e0a32b18b3f1 2024-12-05T07:42:28,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f8a18d9f5f0c44aeb067571c70066666 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f8a18d9f5f0c44aeb067571c70066666 2024-12-05T07:42:28,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/263867a76de440dc8ee0f6d017dbce16 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/263867a76de440dc8ee0f6d017dbce16 2024-12-05T07:42:28,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f753451c958045bcb22b29abd8d8476f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f753451c958045bcb22b29abd8d8476f 2024-12-05T07:42:28,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/37e1694d3358453488ae43d26f5c3f7e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/37e1694d3358453488ae43d26f5c3f7e 2024-12-05T07:42:28,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2a4a4feb75e94abf9de7e32dc60fd254 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2a4a4feb75e94abf9de7e32dc60fd254 2024-12-05T07:42:28,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2231745005204b50a528e6572573a7a2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2231745005204b50a528e6572573a7a2 2024-12-05T07:42:28,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f93e8facd15340bea9417b55d799e7d1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f93e8facd15340bea9417b55d799e7d1 2024-12-05T07:42:28,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6a47bd24af4f4ee4b0ccfcf878f93597 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6a47bd24af4f4ee4b0ccfcf878f93597 2024-12-05T07:42:28,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/46c9c5567b9f441d819e5895ef478d5e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/46c9c5567b9f441d819e5895ef478d5e 2024-12-05T07:42:28,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a48328e8d7c44e4f8d5db859e16c438a to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a48328e8d7c44e4f8d5db859e16c438a 2024-12-05T07:42:28,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7f6ad06b766f46578b3df12252293a5d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/7f6ad06b766f46578b3df12252293a5d 2024-12-05T07:42:28,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6bf37804ca2541dc931b3d0c2af3c10e to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/6bf37804ca2541dc931b3d0c2af3c10e 2024-12-05T07:42:28,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/823d232e27d740be9f0aa5919e4a3439 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/823d232e27d740be9f0aa5919e4a3439 2024-12-05T07:42:28,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a95d04f611954738b4b3b586dd55ae41 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a95d04f611954738b4b3b586dd55ae41 2024-12-05T07:42:28,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/b88cbca5f824453d9b8dca96276418cb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/b88cbca5f824453d9b8dca96276418cb 2024-12-05T07:42:28,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/9ee3e27eb85a432f802718e9e1b79b56 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/9ee3e27eb85a432f802718e9e1b79b56 2024-12-05T07:42:28,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/8d2786ef910041139f6dbe6cc3cb6ffa to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/8d2786ef910041139f6dbe6cc3cb6ffa 2024-12-05T07:42:28,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a25568fa77e44acc9cfb39ef65470785 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/a25568fa77e44acc9cfb39ef65470785 2024-12-05T07:42:28,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f3d6dd4eb2b64a24884e7bd029630eae to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/f3d6dd4eb2b64a24884e7bd029630eae 2024-12-05T07:42:28,398 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/recovered.edits/402.seqid, newMaxSeqId=402, maxSeqId=1 2024-12-05T07:42:28,398 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. 2024-12-05T07:42:28,398 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for 45734b659127c96dcd616039a555b19f: Waiting for close lock at 1733384547025Running coprocessor pre-close hooks at 1733384547025Disabling compacts and flushes for region at 1733384547025Disabling writes for close at 1733384547025Obtaining lock to block concurrent updates at 1733384547025Preparing flush snapshotting stores in 45734b659127c96dcd616039a555b19f at 1733384547025Finished memstore snapshotting TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1733384547026 (+1 ms)Flushing stores of TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f. at 1733384547026Flushing 45734b659127c96dcd616039a555b19f/A: creating writer at 1733384547026Flushing 45734b659127c96dcd616039a555b19f/A: appending metadata at 1733384547029 (+3 ms)Flushing 45734b659127c96dcd616039a555b19f/A: closing flushed file at 1733384547029Flushing 45734b659127c96dcd616039a555b19f/B: creating writer at 1733384547446 (+417 ms)Flushing 45734b659127c96dcd616039a555b19f/B: appending metadata at 1733384547450 (+4 ms)Flushing 45734b659127c96dcd616039a555b19f/B: closing flushed file at 1733384547450Flushing 45734b659127c96dcd616039a555b19f/C: creating writer at 1733384547865 (+415 ms)Flushing 45734b659127c96dcd616039a555b19f/C: appending metadata at 1733384547870 (+5 ms)Flushing 45734b659127c96dcd616039a555b19f/C: closing flushed file at 1733384547871 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@759be2d7: reopening flushed file at 1733384548329 (+458 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a32c312: reopening flushed file at 1733384548333 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32ca12a0: reopening flushed file at 1733384548336 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 45734b659127c96dcd616039a555b19f in 1315ms, sequenceid=399, compaction requested=true at 1733384548340 (+4 ms)Writing region close event to WAL at 1733384548395 (+55 ms)Running coprocessor post-close hooks at 1733384548398 (+3 ms)Closed at 1733384548398 2024-12-05T07:42:28,401 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed 45734b659127c96dcd616039a555b19f 2024-12-05T07:42:28,402 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=45734b659127c96dcd616039a555b19f, regionState=CLOSED 2024-12-05T07:42:28,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:42:28,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-05T07:42:28,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure 45734b659127c96dcd616039a555b19f, server=fc6cd38557f3,36693,1733384394401 in 1.5320 sec 2024-12-05T07:42:28,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-12-05T07:42:28,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45734b659127c96dcd616039a555b19f, UNASSIGN in 1.5370 sec 2024-12-05T07:42:28,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-05T07:42:28,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5410 sec 2024-12-05T07:42:28,410 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384548410"}]},"ts":"1733384548410"} 2024-12-05T07:42:28,412 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T07:42:28,412 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T07:42:28,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5520 sec 2024-12-05T07:42:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T07:42:29,007 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:29,007 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T07:42:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:29,009 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:29,010 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T07:42:29,011 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f 2024-12-05T07:42:29,013 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/recovered.edits] 2024-12-05T07:42:29,015 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5ecff829c2244f678126519f3cddd6bd to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/5ecff829c2244f678126519f3cddd6bd 2024-12-05T07:42:29,016 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/f45f6f18c7d94e63a60d02b5a9048e5f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/f45f6f18c7d94e63a60d02b5a9048e5f 2024-12-05T07:42:29,017 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/fc9351ded1254d88a0f732f890f692d2 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/A/fc9351ded1254d88a0f732f890f692d2 2024-12-05T07:42:29,019 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/532227c255904d50beebbd6e3a31c319 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/532227c255904d50beebbd6e3a31c319 2024-12-05T07:42:29,020 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/64b3131db6be4a6ea9d6608dbc3c611d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/64b3131db6be4a6ea9d6608dbc3c611d 2024-12-05T07:42:29,020 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/c673e54e63b34aec822bcd5a923ad8bb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/B/c673e54e63b34aec822bcd5a923ad8bb 2024-12-05T07:42:29,022 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/1c826a0e89d044c082da981558f48e82 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/1c826a0e89d044c082da981558f48e82 2024-12-05T07:42:29,023 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2e1120bdddbf484493d77f1274fe7c94 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/2e1120bdddbf484493d77f1274fe7c94 2024-12-05T07:42:29,024 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/598eba368a1c4a47a5194d8fbacbd683 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/C/598eba368a1c4a47a5194d8fbacbd683 2024-12-05T07:42:29,026 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/recovered.edits/402.seqid to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f/recovered.edits/402.seqid 2024-12-05T07:42:29,027 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/45734b659127c96dcd616039a555b19f 2024-12-05T07:42:29,027 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T07:42:29,028 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:29,031 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T07:42:29,033 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T07:42:29,034 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:29,034 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T07:42:29,034 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733384549034"}]},"ts":"9223372036854775807"} 2024-12-05T07:42:29,036 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T07:42:29,036 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 45734b659127c96dcd616039a555b19f, NAME => 'TestAcidGuarantees,,1733384515112.45734b659127c96dcd616039a555b19f.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T07:42:29,036 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T07:42:29,037 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733384549036"}]},"ts":"9223372036854775807"} 2024-12-05T07:42:29,039 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T07:42:29,039 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:29,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-12-05T07:42:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T07:42:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T07:42:29,118 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:29,124 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=236 (was 241), OpenFileDescriptor=443 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=356 (was 410), ProcessCount=11 (was 11), AvailableMemoryMB=6412 (was 6726) 2024-12-05T07:42:29,132 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=356, ProcessCount=11, AvailableMemoryMB=6409 2024-12-05T07:42:29,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T07:42:29,133 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T07:42:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:29,135 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T07:42:29,135 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:29,135 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-12-05T07:42:29,136 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T07:42:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-05T07:42:29,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742314_1490 (size=959) 2024-12-05T07:42:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-05T07:42:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-05T07:42:29,545 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15 2024-12-05T07:42:29,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742315_1491 (size=53) 2024-12-05T07:42:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-05T07:42:29,955 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:42:29,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e2b2ca43411f4bcc833039c57163fe52, disabling compactions & flushes 2024-12-05T07:42:29,956 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:29,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:29,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. after waiting 0 ms 2024-12-05T07:42:29,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:29,956 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:29,956 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e2b2ca43411f4bcc833039c57163fe52: Waiting for close lock at 1733384549956Disabling compacts and flushes for region at 1733384549956Disabling writes for close at 1733384549956Writing region close event to WAL at 1733384549956Closed at 1733384549956 2024-12-05T07:42:29,958 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T07:42:29,959 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733384549958"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733384549958"}]},"ts":"1733384549958"} 2024-12-05T07:42:29,962 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T07:42:29,963 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T07:42:29,964 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384549963"}]},"ts":"1733384549963"} 2024-12-05T07:42:29,965 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T07:42:29,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, ASSIGN}] 2024-12-05T07:42:29,967 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, ASSIGN 2024-12-05T07:42:29,968 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, ASSIGN; state=OFFLINE, location=fc6cd38557f3,36693,1733384394401; forceNewPlan=false, retain=false 2024-12-05T07:42:30,119 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=e2b2ca43411f4bcc833039c57163fe52, regionState=OPENING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:30,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, ASSIGN because future has completed 2024-12-05T07:42:30,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:42:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-05T07:42:30,282 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:30,283 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} 2024-12-05T07:42:30,284 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,284 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:42:30,284 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,284 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,286 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,288 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b2ca43411f4bcc833039c57163fe52 columnFamilyName A 2024-12-05T07:42:30,288 DEBUG [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:30,288 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(327): Store=e2b2ca43411f4bcc833039c57163fe52/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:42:30,289 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,290 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b2ca43411f4bcc833039c57163fe52 columnFamilyName B 2024-12-05T07:42:30,290 DEBUG [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:30,291 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(327): Store=e2b2ca43411f4bcc833039c57163fe52/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:42:30,291 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,292 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b2ca43411f4bcc833039c57163fe52 columnFamilyName C 2024-12-05T07:42:30,293 DEBUG [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:30,293 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(327): Store=e2b2ca43411f4bcc833039c57163fe52/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:42:30,293 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,294 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,295 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,297 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,297 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,298 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T07:42:30,300 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,302 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T07:42:30,303 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened e2b2ca43411f4bcc833039c57163fe52; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66805660, jitterRate=-0.004518091678619385}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T07:42:30,303 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:30,304 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for e2b2ca43411f4bcc833039c57163fe52: Running coprocessor pre-open hook at 1733384550284Writing region info on filesystem at 1733384550284Initializing all the Stores at 1733384550285 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384550285Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384550286 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384550286Cleaning up temporary data from old regions at 1733384550297 (+11 ms)Running coprocessor post-open hooks at 1733384550303 (+6 ms)Region opened successfully at 1733384550304 (+1 ms) 2024-12-05T07:42:30,304 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., pid=136, masterSystemTime=1733384550275 2024-12-05T07:42:30,306 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:30,306 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:30,307 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=e2b2ca43411f4bcc833039c57163fe52, regionState=OPEN, openSeqNum=2, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:30,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:42:30,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-05T07:42:30,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 in 188 msec 2024-12-05T07:42:30,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-05T07:42:30,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, ASSIGN in 346 msec 2024-12-05T07:42:30,316 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T07:42:30,316 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384550316"}]},"ts":"1733384550316"} 2024-12-05T07:42:30,318 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T07:42:30,319 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T07:42:30,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-12-05T07:42:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-12-05T07:42:31,287 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:31,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:31,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:42:31,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:42:31,292 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:42:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:42:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:42:31,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60327653, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:42:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:42:31,294 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:31,296 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:42:31,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b16d54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:31,297 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:42:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:42:31,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:42:31,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:42:31,300 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T07:42:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is fc6cd38557f3,33643,1733384393481 2024-12-05T07:42:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1602e816 2024-12-05T07:42:31,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T07:42:31,302 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T07:42:31,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T07:42:31,304 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T07:42:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:31,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742316_1492 (size=995) 2024-12-05T07:42:31,721 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-12-05T07:42:31,721 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-12-05T07:42:31,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T07:42:31,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, REOPEN/MOVE}] 2024-12-05T07:42:31,731 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, REOPEN/MOVE 2024-12-05T07:42:31,732 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=e2b2ca43411f4bcc833039c57163fe52, regionState=CLOSING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:31,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, REOPEN/MOVE because future has completed 2024-12-05T07:42:31,733 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T07:42:31,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:42:31,886 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:31,886 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T07:42:31,886 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing e2b2ca43411f4bcc833039c57163fe52, disabling compactions & flushes 2024-12-05T07:42:31,886 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:31,886 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:31,886 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. after waiting 0 ms 2024-12-05T07:42:31,886 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:31,894 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T07:42:31,895 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:31,895 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for e2b2ca43411f4bcc833039c57163fe52: Waiting for close lock at 1733384551886Running coprocessor pre-close hooks at 1733384551886Disabling compacts and flushes for region at 1733384551886Disabling writes for close at 1733384551886Writing region close event to WAL at 1733384551887 (+1 ms)Running coprocessor post-close hooks at 1733384551895 (+8 ms)Closed at 1733384551895 2024-12-05T07:42:31,895 WARN [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: e2b2ca43411f4bcc833039c57163fe52 to self. 2024-12-05T07:42:31,897 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:31,898 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=e2b2ca43411f4bcc833039c57163fe52, regionState=CLOSED 2024-12-05T07:42:31,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:42:31,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-05T07:42:31,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 in 167 msec 2024-12-05T07:42:31,903 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, REOPEN/MOVE; state=CLOSED, location=fc6cd38557f3,36693,1733384394401; forceNewPlan=false, retain=true 2024-12-05T07:42:32,054 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=e2b2ca43411f4bcc833039c57163fe52, regionState=OPENING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, REOPEN/MOVE because future has completed 2024-12-05T07:42:32,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:42:32,217 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:32,217 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} 2024-12-05T07:42:32,218 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,218 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T07:42:32,218 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,218 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,219 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,220 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b2ca43411f4bcc833039c57163fe52 columnFamilyName A 2024-12-05T07:42:32,222 DEBUG [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:32,223 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(327): Store=e2b2ca43411f4bcc833039c57163fe52/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:42:32,223 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,224 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b2ca43411f4bcc833039c57163fe52 columnFamilyName B 2024-12-05T07:42:32,224 DEBUG [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:32,224 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(327): Store=e2b2ca43411f4bcc833039c57163fe52/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:42:32,224 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,225 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2b2ca43411f4bcc833039c57163fe52 columnFamilyName C 2024-12-05T07:42:32,225 DEBUG [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:32,225 INFO [StoreOpener-e2b2ca43411f4bcc833039c57163fe52-1 {}] regionserver.HStore(327): Store=e2b2ca43411f4bcc833039c57163fe52/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T07:42:32,225 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,226 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,226 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,227 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,227 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,227 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T07:42:32,228 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,229 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened e2b2ca43411f4bcc833039c57163fe52; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70764885, jitterRate=0.0544789582490921}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T07:42:32,229 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,229 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for e2b2ca43411f4bcc833039c57163fe52: Running coprocessor pre-open hook at 1733384552218Writing region info on filesystem at 1733384552218Initializing all the Stores at 1733384552218Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384552218Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384552219 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733384552219Cleaning up temporary data from old regions at 1733384552227 (+8 ms)Running coprocessor post-open hooks at 1733384552229 (+2 ms)Region opened successfully at 1733384552229 2024-12-05T07:42:32,230 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., pid=141, masterSystemTime=1733384552213 2024-12-05T07:42:32,231 DEBUG [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:32,231 INFO [RS_OPEN_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:32,231 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=e2b2ca43411f4bcc833039c57163fe52, regionState=OPEN, openSeqNum=5, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:42:32,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-12-05T07:42:32,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 in 175 msec 2024-12-05T07:42:32,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-05T07:42:32,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, REOPEN/MOVE in 503 msec 2024-12-05T07:42:32,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-05T07:42:32,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-12-05T07:42:32,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 932 msec 2024-12-05T07:42:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T07:42:32,239 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:32,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:42:32,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:42:32,241 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:42:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:42:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:42:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eca70e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:42:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:42:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:32,242 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:42:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25d90f5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,243 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:42:32,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:42:32,244 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ba0ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:42:32,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:32,245 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:42:32,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30906b1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:42:32,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:42:32,246 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:42:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:42:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:42:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615021b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:42:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:42:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:32,248 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:42:32,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ee523b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:42:32,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:42:32,250 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:42:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:42:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:42:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1749579a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:42:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:42:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:32,251 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:42:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b00714b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:42:32,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:42:32,253 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:42:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:42:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:42:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6feddd4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:42:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:42:32,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:32,255 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:42:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192d75d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:42:32,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:42:32,257 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:42:32,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:42:32,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:42:32,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42bed1b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:42:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:42:32,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:32,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:42:32,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27740e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:42:32,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:42:32,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:42:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:42:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:42:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a49f60b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,261 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:42:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:42:32,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:32,263 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:42:32,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61efc180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:42:32,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:42:32,264 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:42:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:42:32,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:42:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28cb23c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:42:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:42:32,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:32,265 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:42:32,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b7fdce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8fc71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:42:32,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:42:32,267 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:42:32,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:42:32,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:42:32,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1be3b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:42:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:42:32,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:32,268 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:42:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219135bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc96918, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fc6cd38557f3,33643,-1 for getting cluster id 2024-12-05T07:42:32,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T07:42:32,270 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ff1984c4-b366-4c5a-a4f7-c150e8d1c04b' 2024-12-05T07:42:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T07:42:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ff1984c4-b366-4c5a-a4f7-c150e8d1c04b" 2024-12-05T07:42:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57f7ad61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fc6cd38557f3,33643,-1] 2024-12-05T07:42:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T07:42:32,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:32,271 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T07:42:32,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eab3c49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T07:42:32,272 DEBUG [Thread-2225 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:42:32,272 DEBUG [Thread-2236 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:42:32,272 DEBUG [Thread-2223 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:42:32,272 DEBUG [Thread-2232 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:42:32,272 DEBUG [Thread-2217 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:42:32,272 DEBUG [Thread-2228 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:42:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:42:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:42:32,272 DEBUG [Thread-2234 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:42:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:42:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:42:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:42:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:42:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:42:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:42:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:42:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:42:32,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:42:32,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:42:32,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:42:32,273 DEBUG [Thread-2219 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:42:32,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:42:32,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:42:32,274 DEBUG [Thread-2221 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:42:32,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:42:32,274 DEBUG [Thread-2230 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T07:42:32,274 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:32,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:42:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:42:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:42:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:42:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:42:32,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-12-05T07:42:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:42:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fc6cd38557f3,36693,1733384394401, seqNum=-1] 2024-12-05T07:42:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:42:32,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T07:42:32,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T07:42:32,276 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:42:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:42:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:42:32,277 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:32,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:42:32,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:42:32,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:42:32,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:42:32,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:42:32,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T07:42:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:42:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:42:32,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:42:32,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:42:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:42:32,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5] 2024-12-05T07:42:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,283 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:32,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T07:42:32,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384612303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057ced2db782e6425891db1f38a331c626_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384552282/Put/seqid=0 2024-12-05T07:42:32,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384612303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384612305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384612306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384612306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742317_1493 (size=12154) 2024-12-05T07:42:32,312 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:32,315 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057ced2db782e6425891db1f38a331c626_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ced2db782e6425891db1f38a331c626_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:32,316 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/0af9690b2dd44e2884518a800861bfd9, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:32,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/0af9690b2dd44e2884518a800861bfd9 is 175, key is test_row_0/A:col10/1733384552282/Put/seqid=0 2024-12-05T07:42:32,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742318_1494 (size=30955) 2024-12-05T07:42:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T07:42:32,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384612415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384612415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384612416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384612416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384612416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,419 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T07:42:32,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:32,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:32,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:32,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T07:42:32,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:32,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:32,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:32,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T07:42:32,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384612625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384612626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384612626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384612626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384612626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,733 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/0af9690b2dd44e2884518a800861bfd9 2024-12-05T07:42:32,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T07:42:32,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:32,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:32,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:32,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/5001887f0cd9431e8777d67bf9b58d1d is 50, key is test_row_0/B:col10/1733384552282/Put/seqid=0 2024-12-05T07:42:32,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742319_1495 (size=12001) 2024-12-05T07:42:32,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T07:42:32,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:32,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:32,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:32,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T07:42:32,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384612936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384612936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384612936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384612937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384612937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:33,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T07:42:33,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:33,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:33,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:33,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/5001887f0cd9431e8777d67bf9b58d1d 2024-12-05T07:42:33,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T07:42:33,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:33,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:33,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:33,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/327d9fb22d234fdab5e4a7ae09e65140 is 50, key is test_row_0/C:col10/1733384552282/Put/seqid=0 2024-12-05T07:42:33,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742320_1496 (size=12001) 2024-12-05T07:42:33,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T07:42:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:33,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:33,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T07:42:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384613446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384613446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384613446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:33,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:33,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384613446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384613446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:33,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T07:42:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:33,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:33,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/327d9fb22d234fdab5e4a7ae09e65140 2024-12-05T07:42:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/0af9690b2dd44e2884518a800861bfd9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/0af9690b2dd44e2884518a800861bfd9 2024-12-05T07:42:33,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/0af9690b2dd44e2884518a800861bfd9, entries=150, sequenceid=17, filesize=30.2 K 2024-12-05T07:42:33,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/5001887f0cd9431e8777d67bf9b58d1d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/5001887f0cd9431e8777d67bf9b58d1d 2024-12-05T07:42:33,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/5001887f0cd9431e8777d67bf9b58d1d, entries=150, sequenceid=17, filesize=11.7 K 2024-12-05T07:42:33,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/327d9fb22d234fdab5e4a7ae09e65140 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/327d9fb22d234fdab5e4a7ae09e65140 2024-12-05T07:42:33,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/327d9fb22d234fdab5e4a7ae09e65140, entries=150, sequenceid=17, filesize=11.7 K 2024-12-05T07:42:33,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e2b2ca43411f4bcc833039c57163fe52 in 1360ms, sequenceid=17, compaction requested=false 2024-12-05T07:42:33,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:33,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-12-05T07:42:33,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:33,655 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-12-05T07:42:33,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120580dd03c6319e4d858b180310f5e9539c_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384552305/Put/seqid=0 2024-12-05T07:42:33,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742321_1497 (size=12154) 2024-12-05T07:42:33,913 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T07:42:34,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:34,099 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120580dd03c6319e4d858b180310f5e9539c_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120580dd03c6319e4d858b180310f5e9539c_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:34,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/1540314b437641508f8a94f78ac4ea56, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:34,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/1540314b437641508f8a94f78ac4ea56 is 175, key is test_row_0/A:col10/1733384552305/Put/seqid=0 2024-12-05T07:42:34,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742322_1498 (size=30955) 2024-12-05T07:42:34,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T07:42:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:34,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:34,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384614466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:34,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384614466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:34,468 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in e2b2ca43411f4bcc833039c57163fe52 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:34,469 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in e2b2ca43411f4bcc833039c57163fe52 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:34,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384614467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:34,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384614468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:34,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384614468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:34,504 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/1540314b437641508f8a94f78ac4ea56 2024-12-05T07:42:34,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/410ece1bdca048f899bc5d4c6ba9ea09 is 50, key is test_row_0/B:col10/1733384552305/Put/seqid=0 2024-12-05T07:42:34,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742323_1499 (size=12001) 2024-12-05T07:42:34,517 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/410ece1bdca048f899bc5d4c6ba9ea09 2024-12-05T07:42:34,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/2ab50168e28f4eb8ac8e8cbc9b22aaf0 is 50, key is test_row_0/C:col10/1733384552305/Put/seqid=0 2024-12-05T07:42:34,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742324_1500 (size=12001) 2024-12-05T07:42:34,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384614576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:34,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:34,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384614576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384614576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:34,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384614786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:34,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384614786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:34,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384614786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:34,927 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/2ab50168e28f4eb8ac8e8cbc9b22aaf0 2024-12-05T07:42:34,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/1540314b437641508f8a94f78ac4ea56 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/1540314b437641508f8a94f78ac4ea56 2024-12-05T07:42:34,933 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/1540314b437641508f8a94f78ac4ea56, entries=150, sequenceid=40, filesize=30.2 K 2024-12-05T07:42:34,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/410ece1bdca048f899bc5d4c6ba9ea09 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/410ece1bdca048f899bc5d4c6ba9ea09 2024-12-05T07:42:34,936 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/410ece1bdca048f899bc5d4c6ba9ea09, entries=150, sequenceid=40, filesize=11.7 K 2024-12-05T07:42:34,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/2ab50168e28f4eb8ac8e8cbc9b22aaf0 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/2ab50168e28f4eb8ac8e8cbc9b22aaf0 2024-12-05T07:42:34,939 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/2ab50168e28f4eb8ac8e8cbc9b22aaf0, entries=150, sequenceid=40, filesize=11.7 K 2024-12-05T07:42:34,940 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e2b2ca43411f4bcc833039c57163fe52 in 1285ms, sequenceid=40, compaction requested=false 2024-12-05T07:42:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-12-05T07:42:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-12-05T07:42:34,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-05T07:42:34,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6640 sec 2024-12-05T07:42:34,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.6700 sec 2024-12-05T07:42:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:35,097 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:35,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T07:42:35,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120537361566794f4f149a6aeb92b58e4eb6_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384554463/Put/seqid=0 2024-12-05T07:42:35,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742325_1501 (size=12154) 2024-12-05T07:42:35,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384615117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384615120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384615120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384615226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384615226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384615226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384615436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384615436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384615436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,509 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:35,511 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120537361566794f4f149a6aeb92b58e4eb6_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120537361566794f4f149a6aeb92b58e4eb6_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:35,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/3c13471d22374dffa6248e3f19648d7f, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:35,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/3c13471d22374dffa6248e3f19648d7f is 175, key is test_row_0/A:col10/1733384554463/Put/seqid=0 2024-12-05T07:42:35,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742326_1502 (size=30955) 2024-12-05T07:42:35,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384615745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384615745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384615746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:35,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:35,916 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/3c13471d22374dffa6248e3f19648d7f 2024-12-05T07:42:35,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/b74413cc31ea4779b76b1b405e1f57ef is 50, key is test_row_0/B:col10/1733384554463/Put/seqid=0 2024-12-05T07:42:35,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742327_1503 (size=12001) 2024-12-05T07:42:36,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384616256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:36,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384616256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:36,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384616256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:36,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:36,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:36,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:36,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/b74413cc31ea4779b76b1b405e1f57ef 2024-12-05T07:42:36,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/d7245618fd1942dabf5fc0f53541d8b8 is 50, key is test_row_0/C:col10/1733384554463/Put/seqid=0 2024-12-05T07:42:36,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742328_1504 (size=12001) 2024-12-05T07:42:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-05T07:42:36,446 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:36,446 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-05T07:42:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T07:42:36,448 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:36,448 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:36,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:36,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384616486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:36,488 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in e2b2ca43411f4bcc833039c57163fe52 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:36,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384616495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:36,497 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in e2b2ca43411f4bcc833039c57163fe52 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:36,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T07:42:36,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-05T07:42:36,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:36,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:36,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:36,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:36,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:36,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/d7245618fd1942dabf5fc0f53541d8b8 2024-12-05T07:42:36,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/3c13471d22374dffa6248e3f19648d7f as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/3c13471d22374dffa6248e3f19648d7f 2024-12-05T07:42:36,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/3c13471d22374dffa6248e3f19648d7f, entries=150, sequenceid=55, filesize=30.2 K 2024-12-05T07:42:36,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/b74413cc31ea4779b76b1b405e1f57ef as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/b74413cc31ea4779b76b1b405e1f57ef 2024-12-05T07:42:36,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/b74413cc31ea4779b76b1b405e1f57ef, entries=150, sequenceid=55, filesize=11.7 K 2024-12-05T07:42:36,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/d7245618fd1942dabf5fc0f53541d8b8 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/d7245618fd1942dabf5fc0f53541d8b8 2024-12-05T07:42:36,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/d7245618fd1942dabf5fc0f53541d8b8, entries=150, sequenceid=55, filesize=11.7 K 2024-12-05T07:42:36,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e2b2ca43411f4bcc833039c57163fe52 in 1647ms, sequenceid=55, compaction requested=true 2024-12-05T07:42:36,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:36,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:36,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:36,744 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:36,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:36,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:36,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:36,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T07:42:36,745 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:36,745 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/A is initiating minor compaction (all files) 2024-12-05T07:42:36,745 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/A in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:36,745 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/0af9690b2dd44e2884518a800861bfd9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/1540314b437641508f8a94f78ac4ea56, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/3c13471d22374dffa6248e3f19648d7f] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=90.7 K 2024-12-05T07:42:36,745 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:36,745 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/0af9690b2dd44e2884518a800861bfd9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/1540314b437641508f8a94f78ac4ea56, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/3c13471d22374dffa6248e3f19648d7f] 2024-12-05T07:42:36,745 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0af9690b2dd44e2884518a800861bfd9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733384552282 2024-12-05T07:42:36,746 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1540314b437641508f8a94f78ac4ea56, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733384552299 2024-12-05T07:42:36,746 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c13471d22374dffa6248e3f19648d7f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733384554463 2024-12-05T07:42:36,750 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:36,751 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412052fd580235e864d7d9028322059ec6cdd_e2b2ca43411f4bcc833039c57163fe52 store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:36,753 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412052fd580235e864d7d9028322059ec6cdd_e2b2ca43411f4bcc833039c57163fe52, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:36,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-05T07:42:36,753 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052fd580235e864d7d9028322059ec6cdd_e2b2ca43411f4bcc833039c57163fe52 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:36,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:36,753 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-12-05T07:42:36,754 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:36,755 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:36,755 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/B is initiating minor compaction (all files) 2024-12-05T07:42:36,755 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/B in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:36,755 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/5001887f0cd9431e8777d67bf9b58d1d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/410ece1bdca048f899bc5d4c6ba9ea09, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/b74413cc31ea4779b76b1b405e1f57ef] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=35.2 K 2024-12-05T07:42:36,755 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5001887f0cd9431e8777d67bf9b58d1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733384552282 2024-12-05T07:42:36,756 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 410ece1bdca048f899bc5d4c6ba9ea09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733384552299 2024-12-05T07:42:36,756 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting b74413cc31ea4779b76b1b405e1f57ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733384554463 2024-12-05T07:42:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T07:42:36,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742329_1505 (size=4469) 2024-12-05T07:42:36,770 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:36,770 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/a1a6f1100a9c4cf5ac1f28afc1f64c61 is 50, key is test_row_0/B:col10/1733384554463/Put/seqid=0 2024-12-05T07:42:36,771 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#A#compaction#428 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:36,771 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/a6717b18765043339be8e2905f09b8ed is 175, key is test_row_0/A:col10/1733384554463/Put/seqid=0 2024-12-05T07:42:36,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059108ea12e64c44348226e51d38907f07_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384555110/Put/seqid=0 2024-12-05T07:42:36,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742331_1507 (size=12104) 2024-12-05T07:42:36,783 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/a1a6f1100a9c4cf5ac1f28afc1f64c61 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/a1a6f1100a9c4cf5ac1f28afc1f64c61 2024-12-05T07:42:36,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742332_1508 (size=12154) 2024-12-05T07:42:36,787 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/B of e2b2ca43411f4bcc833039c57163fe52 into a1a6f1100a9c4cf5ac1f28afc1f64c61(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:36,787 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:36,788 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/B, priority=13, startTime=1733384556744; duration=0sec 2024-12-05T07:42:36,788 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:36,788 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:B 2024-12-05T07:42:36,788 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:36,789 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:36,789 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/C is initiating minor compaction (all files) 2024-12-05T07:42:36,789 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/C in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:36,789 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/327d9fb22d234fdab5e4a7ae09e65140, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/2ab50168e28f4eb8ac8e8cbc9b22aaf0, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/d7245618fd1942dabf5fc0f53541d8b8] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=35.2 K 2024-12-05T07:42:36,789 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 327d9fb22d234fdab5e4a7ae09e65140, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733384552282 2024-12-05T07:42:36,789 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ab50168e28f4eb8ac8e8cbc9b22aaf0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1733384552299 2024-12-05T07:42:36,789 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting d7245618fd1942dabf5fc0f53541d8b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733384554463 2024-12-05T07:42:36,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742330_1506 (size=31058) 2024-12-05T07:42:36,795 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#C#compaction#431 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:36,796 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/e0842fb53ffc423c9635fda16d9091ba is 50, key is test_row_0/C:col10/1733384554463/Put/seqid=0 2024-12-05T07:42:36,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742333_1509 (size=12104) 2024-12-05T07:42:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T07:42:37,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:37,192 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059108ea12e64c44348226e51d38907f07_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059108ea12e64c44348226e51d38907f07_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:37,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/bc66f0ebc99f4624b9bff773e23cae6b, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:37,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/bc66f0ebc99f4624b9bff773e23cae6b is 175, key is test_row_0/A:col10/1733384555110/Put/seqid=0 2024-12-05T07:42:37,198 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/a6717b18765043339be8e2905f09b8ed as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/a6717b18765043339be8e2905f09b8ed 2024-12-05T07:42:37,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742334_1510 (size=30955) 2024-12-05T07:42:37,203 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/A of e2b2ca43411f4bcc833039c57163fe52 into a6717b18765043339be8e2905f09b8ed(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:37,203 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:37,203 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/A, priority=13, startTime=1733384556744; duration=0sec 2024-12-05T07:42:37,203 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:37,203 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:A 2024-12-05T07:42:37,209 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/e0842fb53ffc423c9635fda16d9091ba as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/e0842fb53ffc423c9635fda16d9091ba 2024-12-05T07:42:37,213 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/C of e2b2ca43411f4bcc833039c57163fe52 into e0842fb53ffc423c9635fda16d9091ba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:37,213 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:37,213 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/C, priority=13, startTime=1733384556744; duration=0sec 2024-12-05T07:42:37,213 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:37,213 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:C 2024-12-05T07:42:37,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:37,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:37,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384617281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:37,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384617282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:37,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384617283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:37,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384617386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:37,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384617396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384617396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T07:42:37,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384617596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:37,601 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/bc66f0ebc99f4624b9bff773e23cae6b 2024-12-05T07:42:37,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e2872981610c454fafd1644d673a4786 is 50, key is test_row_0/B:col10/1733384555110/Put/seqid=0 2024-12-05T07:42:37,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384617606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384617606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:37,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742335_1511 (size=12001) 2024-12-05T07:42:37,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384617905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:37,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384617916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:37,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384617916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:37,918 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:38,011 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e2872981610c454fafd1644d673a4786 2024-12-05T07:42:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/40291d4884ca4c988b537d7b610b354b is 50, key is test_row_0/C:col10/1733384555110/Put/seqid=0 2024-12-05T07:42:38,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742336_1512 (size=12001) 2024-12-05T07:42:38,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:38,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384618416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:38,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384618425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:38,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:38,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384618426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:38,432 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/40291d4884ca4c988b537d7b610b354b 2024-12-05T07:42:38,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/bc66f0ebc99f4624b9bff773e23cae6b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bc66f0ebc99f4624b9bff773e23cae6b 2024-12-05T07:42:38,438 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bc66f0ebc99f4624b9bff773e23cae6b, entries=150, sequenceid=76, filesize=30.2 K 2024-12-05T07:42:38,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e2872981610c454fafd1644d673a4786 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e2872981610c454fafd1644d673a4786 2024-12-05T07:42:38,441 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e2872981610c454fafd1644d673a4786, entries=150, sequenceid=76, filesize=11.7 K 2024-12-05T07:42:38,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/40291d4884ca4c988b537d7b610b354b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/40291d4884ca4c988b537d7b610b354b 2024-12-05T07:42:38,444 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/40291d4884ca4c988b537d7b610b354b, entries=150, sequenceid=76, filesize=11.7 K 2024-12-05T07:42:38,445 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for e2b2ca43411f4bcc833039c57163fe52 in 1692ms, sequenceid=76, compaction requested=false 2024-12-05T07:42:38,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:38,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:38,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-05T07:42:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-05T07:42:38,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-05T07:42:38,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9980 sec 2024-12-05T07:42:38,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.0020 sec 2024-12-05T07:42:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-05T07:42:38,596 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:38,596 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-05T07:42:38,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T07:42:38,598 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:38,599 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:38,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:38,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T07:42:38,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T07:42:38,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:38,751 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-12-05T07:42:38,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d0864854835342faab8fbf540c4f62b7_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384557273/Put/seqid=0 2024-12-05T07:42:38,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742337_1513 (size=12154) 2024-12-05T07:42:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T07:42:39,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:39,162 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d0864854835342faab8fbf540c4f62b7_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d0864854835342faab8fbf540c4f62b7_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7d9b61730160449b9f1b4cb0b1b77f57, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:39,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7d9b61730160449b9f1b4cb0b1b77f57 is 175, key is test_row_0/A:col10/1733384557273/Put/seqid=0 2024-12-05T07:42:39,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742338_1514 (size=30955) 2024-12-05T07:42:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T07:42:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:39,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:39,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384619459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:39,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384619460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:39,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384619461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:39,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384619566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:39,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384619566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:39,571 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7d9b61730160449b9f1b4cb0b1b77f57 2024-12-05T07:42:39,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:39,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384619576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:39,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/6f1bc2b066194b3082496e6c670d3151 is 50, key is test_row_0/B:col10/1733384557273/Put/seqid=0 2024-12-05T07:42:39,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742339_1515 (size=12001) 2024-12-05T07:42:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T07:42:39,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384619776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:39,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384619776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:39,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:39,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:39,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384619786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,012 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/6f1bc2b066194b3082496e6c670d3151 2024-12-05T07:42:40,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/4524f83726a3454ab84bdef38be45dcc is 50, key is test_row_0/C:col10/1733384557273/Put/seqid=0 2024-12-05T07:42:40,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742340_1516 (size=12001) 2024-12-05T07:42:40,041 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/4524f83726a3454ab84bdef38be45dcc 2024-12-05T07:42:40,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7d9b61730160449b9f1b4cb0b1b77f57 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7d9b61730160449b9f1b4cb0b1b77f57 2024-12-05T07:42:40,056 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7d9b61730160449b9f1b4cb0b1b77f57, entries=150, sequenceid=94, filesize=30.2 K 2024-12-05T07:42:40,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/6f1bc2b066194b3082496e6c670d3151 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6f1bc2b066194b3082496e6c670d3151 2024-12-05T07:42:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,059 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6f1bc2b066194b3082496e6c670d3151, entries=150, sequenceid=94, filesize=11.7 K 2024-12-05T07:42:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/4524f83726a3454ab84bdef38be45dcc as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/4524f83726a3454ab84bdef38be45dcc 2024-12-05T07:42:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,063 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/4524f83726a3454ab84bdef38be45dcc, entries=150, sequenceid=94, filesize=11.7 K 2024-12-05T07:42:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,063 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e2b2ca43411f4bcc833039c57163fe52 in 1312ms, sequenceid=94, compaction requested=true 2024-12-05T07:42:40,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:40,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-05T07:42:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-05T07:42:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-05T07:42:40,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4660 sec 2024-12-05T07:42:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.4710 sec 2024-12-05T07:42:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:40,093 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:40,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-12-05T07:42:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051cc1adaf5af54abebe8a491734fe72da_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384559460/Put/seqid=0 2024-12-05T07:42:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742341_1517 (size=12154) 2024-12-05T07:42:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,106 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:40,108 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051cc1adaf5af54abebe8a491734fe72da_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051cc1adaf5af54abebe8a491734fe72da_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:40,109 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/e223280cc3fa4fc7b6b5612a4f1bb412, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:40,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/e223280cc3fa4fc7b6b5612a4f1bb412 is 175, key is test_row_0/A:col10/1733384559460/Put/seqid=0 2024-12-05T07:42:40,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742342_1518 (size=30951) 2024-12-05T07:42:40,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384620118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384620119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384620121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384620226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384620226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384620236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384620435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384620435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384620446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384620496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,498 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_0' in e2b2ca43411f4bcc833039c57163fe52 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384620505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,509 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_1' in e2b2ca43411f4bcc833039c57163fe52 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:40,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,516 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/e223280cc3fa4fc7b6b5612a4f1bb412 2024-12-05T07:42:40,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/ae35f3fdd875482ab23c60fa800a7ceb is 50, key is test_row_0/B:col10/1733384559460/Put/seqid=0 2024-12-05T07:42:40,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742343_1519 (size=9657) 2024-12-05T07:42:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T07:42:40,747 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:40,748 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:40,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384620747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384620747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-05T07:42:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T07:42:40,750 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:40,751 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:40,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384620757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T07:42:40,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T07:42:40,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:40,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:40,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:40,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:40,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:40,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/ae35f3fdd875482ab23c60fa800a7ceb 2024-12-05T07:42:40,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/8dc2ae0ca06341968316d731666375ea is 50, key is test_row_0/C:col10/1733384559460/Put/seqid=0 2024-12-05T07:42:40,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742344_1520 (size=9657) 2024-12-05T07:42:41,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T07:42:41,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:41,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:41,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:41,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:41,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T07:42:41,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T07:42:41,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:41,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:41,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:41,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:41,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:41,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:41,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384621256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384621256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:41,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:41,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384621266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:41,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:41,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:41,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:41,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/8dc2ae0ca06341968316d731666375ea 2024-12-05T07:42:41,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/e223280cc3fa4fc7b6b5612a4f1bb412 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e223280cc3fa4fc7b6b5612a4f1bb412 2024-12-05T07:42:41,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e223280cc3fa4fc7b6b5612a4f1bb412, entries=150, sequenceid=116, filesize=30.2 K 2024-12-05T07:42:41,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/ae35f3fdd875482ab23c60fa800a7ceb as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/ae35f3fdd875482ab23c60fa800a7ceb 2024-12-05T07:42:41,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/ae35f3fdd875482ab23c60fa800a7ceb, entries=100, sequenceid=116, filesize=9.4 K 2024-12-05T07:42:41,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/8dc2ae0ca06341968316d731666375ea as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8dc2ae0ca06341968316d731666375ea 2024-12-05T07:42:41,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8dc2ae0ca06341968316d731666375ea, entries=100, sequenceid=116, filesize=9.4 K 2024-12-05T07:42:41,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e2b2ca43411f4bcc833039c57163fe52 in 1267ms, sequenceid=116, compaction requested=true 2024-12-05T07:42:41,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:41,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:41,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:41,361 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:42:41,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:41,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:41,361 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:42:41,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:41,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:41,362 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123919 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:42:41,362 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:42:41,362 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/B is initiating minor compaction (all files) 2024-12-05T07:42:41,362 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/A is initiating minor compaction (all files) 2024-12-05T07:42:41,362 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/B in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:41,362 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/A in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:41,362 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/a1a6f1100a9c4cf5ac1f28afc1f64c61, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e2872981610c454fafd1644d673a4786, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6f1bc2b066194b3082496e6c670d3151, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/ae35f3fdd875482ab23c60fa800a7ceb] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=44.7 K 2024-12-05T07:42:41,362 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/a6717b18765043339be8e2905f09b8ed, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bc66f0ebc99f4624b9bff773e23cae6b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7d9b61730160449b9f1b4cb0b1b77f57, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e223280cc3fa4fc7b6b5612a4f1bb412] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=121.0 K 2024-12-05T07:42:41,362 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:41,362 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/a6717b18765043339be8e2905f09b8ed, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bc66f0ebc99f4624b9bff773e23cae6b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7d9b61730160449b9f1b4cb0b1b77f57, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e223280cc3fa4fc7b6b5612a4f1bb412] 2024-12-05T07:42:41,362 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting a1a6f1100a9c4cf5ac1f28afc1f64c61, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733384554463 2024-12-05T07:42:41,362 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6717b18765043339be8e2905f09b8ed, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733384554463 2024-12-05T07:42:41,363 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc66f0ebc99f4624b9bff773e23cae6b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733384555110 2024-12-05T07:42:41,363 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting e2872981610c454fafd1644d673a4786, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733384555110 2024-12-05T07:42:41,363 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d9b61730160449b9f1b4cb0b1b77f57, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733384557273 2024-12-05T07:42:41,363 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1bc2b066194b3082496e6c670d3151, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733384557273 2024-12-05T07:42:41,363 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e223280cc3fa4fc7b6b5612a4f1bb412, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733384559454 2024-12-05T07:42:41,363 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting ae35f3fdd875482ab23c60fa800a7ceb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733384559460 2024-12-05T07:42:41,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T07:42:41,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:41,367 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-12-05T07:42:41,368 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:41,370 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:41,371 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/f10a6930c5c443c1b5b32c7189c0a684 is 50, key is test_row_0/B:col10/1733384559460/Put/seqid=0 2024-12-05T07:42:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T07:42:41,384 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205c7482b4ade4546899d23082af92edba3_e2b2ca43411f4bcc833039c57163fe52 store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:41,385 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205c7482b4ade4546899d23082af92edba3_e2b2ca43411f4bcc833039c57163fe52, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:41,385 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c7482b4ade4546899d23082af92edba3_e2b2ca43411f4bcc833039c57163fe52 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:41,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742345_1521 (size=12241) 2024-12-05T07:42:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120533d790d432d14184aabd266c42a16b43_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384560114/Put/seqid=0 2024-12-05T07:42:41,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742346_1522 (size=4469) 2024-12-05T07:42:41,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742347_1523 (size=12204) 2024-12-05T07:42:41,804 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/f10a6930c5c443c1b5b32c7189c0a684 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/f10a6930c5c443c1b5b32c7189c0a684 2024-12-05T07:42:41,812 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/B of e2b2ca43411f4bcc833039c57163fe52 into f10a6930c5c443c1b5b32c7189c0a684(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:41,812 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:41,812 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/B, priority=12, startTime=1733384561361; duration=0sec 2024-12-05T07:42:41,812 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:41,812 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:B 2024-12-05T07:42:41,813 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T07:42:41,813 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T07:42:41,814 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/C is initiating minor compaction (all files) 2024-12-05T07:42:41,814 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/C in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:41,814 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/e0842fb53ffc423c9635fda16d9091ba, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/40291d4884ca4c988b537d7b610b354b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/4524f83726a3454ab84bdef38be45dcc, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8dc2ae0ca06341968316d731666375ea] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=44.7 K 2024-12-05T07:42:41,814 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting e0842fb53ffc423c9635fda16d9091ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733384554463 2024-12-05T07:42:41,814 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 40291d4884ca4c988b537d7b610b354b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1733384555110 2024-12-05T07:42:41,814 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 4524f83726a3454ab84bdef38be45dcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733384557273 2024-12-05T07:42:41,815 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc2ae0ca06341968316d731666375ea, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733384559460 2024-12-05T07:42:41,817 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#A#compaction#440 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:41,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:41,817 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/519e72c2cc024bdd80f47f27929d13e1 is 175, key is test_row_0/A:col10/1733384559460/Put/seqid=0 2024-12-05T07:42:41,821 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120533d790d432d14184aabd266c42a16b43_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120533d790d432d14184aabd266c42a16b43_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:41,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/bddb8d2e5bd54d8b9fd96e27c9ba2779, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:41,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/bddb8d2e5bd54d8b9fd96e27c9ba2779 is 175, key is test_row_0/A:col10/1733384560114/Put/seqid=0 2024-12-05T07:42:41,841 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#C#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:41,841 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/beb8fc0f727f4d5ea593f674333298d5 is 50, key is test_row_0/C:col10/1733384559460/Put/seqid=0 2024-12-05T07:42:41,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742348_1524 (size=31302) 2024-12-05T07:42:41,852 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/519e72c2cc024bdd80f47f27929d13e1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/519e72c2cc024bdd80f47f27929d13e1 2024-12-05T07:42:41,858 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/A of e2b2ca43411f4bcc833039c57163fe52 into 519e72c2cc024bdd80f47f27929d13e1(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:41,858 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:41,858 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/A, priority=12, startTime=1733384561361; duration=0sec 2024-12-05T07:42:41,858 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:41,858 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:A 2024-12-05T07:42:41,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742349_1525 (size=31005) 2024-12-05T07:42:41,868 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/bddb8d2e5bd54d8b9fd96e27c9ba2779 2024-12-05T07:42:41,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742350_1526 (size=12241) 2024-12-05T07:42:41,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e7be8d26f86d4b2ca8ca2ffe1e97ce33 is 50, key is test_row_0/B:col10/1733384560114/Put/seqid=0 2024-12-05T07:42:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T07:42:41,886 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/beb8fc0f727f4d5ea593f674333298d5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/beb8fc0f727f4d5ea593f674333298d5 2024-12-05T07:42:41,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742351_1527 (size=12051) 2024-12-05T07:42:41,892 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/C of e2b2ca43411f4bcc833039c57163fe52 into beb8fc0f727f4d5ea593f674333298d5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:41,892 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:41,892 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/C, priority=12, startTime=1733384561361; duration=0sec 2024-12-05T07:42:41,892 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:41,892 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:C 2024-12-05T07:42:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:42,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:42,290 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e7be8d26f86d4b2ca8ca2ffe1e97ce33 2024-12-05T07:42:42,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384622291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:42,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/69a447a47a9c4ceb96015414209075a5 is 50, key is test_row_0/C:col10/1733384560114/Put/seqid=0 2024-12-05T07:42:42,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384622294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384622293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:42,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742352_1528 (size=12051) 2024-12-05T07:42:42,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384622396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:42,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384622405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384622406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:42,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384622606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:42,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384622616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:42,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384622616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:42,699 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/69a447a47a9c4ceb96015414209075a5 2024-12-05T07:42:42,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/bddb8d2e5bd54d8b9fd96e27c9ba2779 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bddb8d2e5bd54d8b9fd96e27c9ba2779 2024-12-05T07:42:42,706 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bddb8d2e5bd54d8b9fd96e27c9ba2779, entries=150, sequenceid=130, filesize=30.3 K 2024-12-05T07:42:42,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e7be8d26f86d4b2ca8ca2ffe1e97ce33 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e7be8d26f86d4b2ca8ca2ffe1e97ce33 2024-12-05T07:42:42,709 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e7be8d26f86d4b2ca8ca2ffe1e97ce33, entries=150, sequenceid=130, filesize=11.8 K 2024-12-05T07:42:42,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/69a447a47a9c4ceb96015414209075a5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/69a447a47a9c4ceb96015414209075a5 2024-12-05T07:42:42,713 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/69a447a47a9c4ceb96015414209075a5, entries=150, sequenceid=130, filesize=11.8 K 2024-12-05T07:42:42,714 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for e2b2ca43411f4bcc833039c57163fe52 in 1347ms, sequenceid=130, compaction requested=false 2024-12-05T07:42:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-05T07:42:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-05T07:42:42,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-05T07:42:42,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9640 sec 2024-12-05T07:42:42,719 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.9700 sec 2024-12-05T07:42:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T07:42:42,896 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:42,896 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-05T07:42:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T07:42:42,897 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:42,898 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:42,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:42,918 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:42,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-12-05T07:42:42,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120535a90fd407b64b06a4bc96a116bcf8b3_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384562292/Put/seqid=0 2024-12-05T07:42:42,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742353_1529 (size=14794) 2024-12-05T07:42:42,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384622932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,933 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:42,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384622932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:42,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384622933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:42,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:42,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T07:42:43,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384623036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:43,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384623036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:43,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384623045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:43,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T07:42:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T07:42:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:43,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T07:42:43,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384623246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:43,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:43,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384623246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:43,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384623255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:43,327 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:43,330 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120535a90fd407b64b06a4bc96a116bcf8b3_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120535a90fd407b64b06a4bc96a116bcf8b3_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:43,331 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/20739e55219a4ed093551f8d48dc80b1, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:43,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/20739e55219a4ed093551f8d48dc80b1 is 175, key is test_row_0/A:col10/1733384562292/Put/seqid=0 2024-12-05T07:42:43,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742354_1530 (size=39749) 2024-12-05T07:42:43,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T07:42:43,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:43,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T07:42:43,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:43,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T07:42:43,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384623556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:43,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384623556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:43,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:43,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:43,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384623566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:43,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T07:42:43,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:43,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,734 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/20739e55219a4ed093551f8d48dc80b1 2024-12-05T07:42:43,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/cf464fee7024413788a9be56c1df3fae is 50, key is test_row_0/B:col10/1733384562292/Put/seqid=0 2024-12-05T07:42:43,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742355_1531 (size=12151) 2024-12-05T07:42:43,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T07:42:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T07:42:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:43,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T07:42:44,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384624066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:44,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384624066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:44,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384624076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:44,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T07:42:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:44,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:44,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/cf464fee7024413788a9be56c1df3fae 2024-12-05T07:42:44,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/c0dc21b578ee424fabe9d71d24c3561c is 50, key is test_row_0/C:col10/1733384562292/Put/seqid=0 2024-12-05T07:42:44,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742356_1532 (size=12151) 2024-12-05T07:42:44,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/c0dc21b578ee424fabe9d71d24c3561c 2024-12-05T07:42:44,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/20739e55219a4ed093551f8d48dc80b1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/20739e55219a4ed093551f8d48dc80b1 2024-12-05T07:42:44,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/20739e55219a4ed093551f8d48dc80b1, entries=200, sequenceid=157, filesize=38.8 K 2024-12-05T07:42:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/cf464fee7024413788a9be56c1df3fae as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/cf464fee7024413788a9be56c1df3fae 2024-12-05T07:42:44,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/cf464fee7024413788a9be56c1df3fae, entries=150, sequenceid=157, filesize=11.9 K 2024-12-05T07:42:44,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/c0dc21b578ee424fabe9d71d24c3561c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/c0dc21b578ee424fabe9d71d24c3561c 2024-12-05T07:42:44,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/c0dc21b578ee424fabe9d71d24c3561c, entries=150, sequenceid=157, filesize=11.9 K 2024-12-05T07:42:44,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e2b2ca43411f4bcc833039c57163fe52 in 1268ms, sequenceid=157, compaction requested=true 2024-12-05T07:42:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:44,186 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:44,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:44,186 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:44,187 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102056 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:44,187 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/A is initiating minor compaction (all files) 2024-12-05T07:42:44,187 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:44,187 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/A in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:44,187 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/B is initiating minor compaction (all files) 2024-12-05T07:42:44,187 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/B in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:44,187 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/519e72c2cc024bdd80f47f27929d13e1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bddb8d2e5bd54d8b9fd96e27c9ba2779, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/20739e55219a4ed093551f8d48dc80b1] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=99.7 K 2024-12-05T07:42:44,187 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:44,187 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/f10a6930c5c443c1b5b32c7189c0a684, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e7be8d26f86d4b2ca8ca2ffe1e97ce33, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/cf464fee7024413788a9be56c1df3fae] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=35.6 K 2024-12-05T07:42:44,187 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/519e72c2cc024bdd80f47f27929d13e1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bddb8d2e5bd54d8b9fd96e27c9ba2779, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/20739e55219a4ed093551f8d48dc80b1] 2024-12-05T07:42:44,187 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting f10a6930c5c443c1b5b32c7189c0a684, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733384557277 2024-12-05T07:42:44,188 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 519e72c2cc024bdd80f47f27929d13e1, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733384557277 2024-12-05T07:42:44,188 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting e7be8d26f86d4b2ca8ca2ffe1e97ce33, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733384560114 2024-12-05T07:42:44,188 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting bddb8d2e5bd54d8b9fd96e27c9ba2779, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733384560114 2024-12-05T07:42:44,188 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting cf464fee7024413788a9be56c1df3fae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733384562290 2024-12-05T07:42:44,188 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20739e55219a4ed093551f8d48dc80b1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733384562287 2024-12-05T07:42:44,194 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:44,195 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:44,196 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/d51b55f08c3347a6aa810f8261473b59 is 50, key is test_row_0/B:col10/1733384562292/Put/seqid=0 2024-12-05T07:42:44,207 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412056e87ea97d02040e4840d272467d99dfc_e2b2ca43411f4bcc833039c57163fe52 store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:44,208 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412056e87ea97d02040e4840d272467d99dfc_e2b2ca43411f4bcc833039c57163fe52, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:44,208 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412056e87ea97d02040e4840d272467d99dfc_e2b2ca43411f4bcc833039c57163fe52 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:44,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742357_1533 (size=12493) 2024-12-05T07:42:44,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742358_1534 (size=4469) 2024-12-05T07:42:44,221 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#A#compaction#449 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:44,221 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/fcb1a94f8fe847bda8045d6817a658e3 is 175, key is test_row_0/A:col10/1733384562292/Put/seqid=0 2024-12-05T07:42:44,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742359_1535 (size=31447) 2024-12-05T07:42:44,227 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/fcb1a94f8fe847bda8045d6817a658e3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/fcb1a94f8fe847bda8045d6817a658e3 2024-12-05T07:42:44,231 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/A of e2b2ca43411f4bcc833039c57163fe52 into fcb1a94f8fe847bda8045d6817a658e3(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:44,231 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:44,231 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/A, priority=13, startTime=1733384564186; duration=0sec 2024-12-05T07:42:44,231 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:44,231 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:A 2024-12-05T07:42:44,231 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:44,232 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:44,232 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/C is initiating minor compaction (all files) 2024-12-05T07:42:44,232 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/C in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:44,232 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/beb8fc0f727f4d5ea593f674333298d5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/69a447a47a9c4ceb96015414209075a5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/c0dc21b578ee424fabe9d71d24c3561c] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=35.6 K 2024-12-05T07:42:44,232 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting beb8fc0f727f4d5ea593f674333298d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1733384557277 2024-12-05T07:42:44,233 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69a447a47a9c4ceb96015414209075a5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733384560114 2024-12-05T07:42:44,233 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0dc21b578ee424fabe9d71d24c3561c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733384562290 2024-12-05T07:42:44,237 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:44,238 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/d4c3588c5b0647dc810ff088e238aeb3 is 50, key is test_row_0/C:col10/1733384562292/Put/seqid=0 2024-12-05T07:42:44,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742360_1536 (size=12493) 2024-12-05T07:42:44,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T07:42:44,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:44,270 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-12-05T07:42:44,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ce8080f07724451fb349f95959de042b_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384562931/Put/seqid=0 2024-12-05T07:42:44,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742361_1537 (size=12304) 2024-12-05T07:42:44,623 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/d51b55f08c3347a6aa810f8261473b59 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/d51b55f08c3347a6aa810f8261473b59 2024-12-05T07:42:44,627 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/B of e2b2ca43411f4bcc833039c57163fe52 into d51b55f08c3347a6aa810f8261473b59(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:44,627 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:44,627 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/B, priority=13, startTime=1733384564186; duration=0sec 2024-12-05T07:42:44,627 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:44,627 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:B 2024-12-05T07:42:44,644 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/d4c3588c5b0647dc810ff088e238aeb3 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/d4c3588c5b0647dc810ff088e238aeb3 2024-12-05T07:42:44,647 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/C of e2b2ca43411f4bcc833039c57163fe52 into d4c3588c5b0647dc810ff088e238aeb3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:44,647 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:44,647 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/C, priority=13, startTime=1733384564186; duration=0sec 2024-12-05T07:42:44,647 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:44,647 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:C 2024-12-05T07:42:44,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:44,687 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ce8080f07724451fb349f95959de042b_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ce8080f07724451fb349f95959de042b_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:44,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7726aa0d54b94a279db0fb6a24a00598, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:44,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7726aa0d54b94a279db0fb6a24a00598 is 175, key is test_row_0/A:col10/1733384562931/Put/seqid=0 2024-12-05T07:42:44,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742362_1538 (size=31105) 2024-12-05T07:42:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T07:42:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:45,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:45,091 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7726aa0d54b94a279db0fb6a24a00598 2024-12-05T07:42:45,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/c9ee93d0691943fa9c368d3c2614e684 is 50, key is test_row_0/B:col10/1733384562931/Put/seqid=0 2024-12-05T07:42:45,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742363_1539 (size=12151) 2024-12-05T07:42:45,100 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/c9ee93d0691943fa9c368d3c2614e684 2024-12-05T07:42:45,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/239bdf255c2c46a0a12869fbee6ba037 is 50, key is test_row_0/C:col10/1733384562931/Put/seqid=0 2024-12-05T07:42:45,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742364_1540 (size=12151) 2024-12-05T07:42:45,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384625108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384625109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384625112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384625216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384625216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384625226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384625425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384625426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384625436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,508 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/239bdf255c2c46a0a12869fbee6ba037 2024-12-05T07:42:45,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7726aa0d54b94a279db0fb6a24a00598 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7726aa0d54b94a279db0fb6a24a00598 2024-12-05T07:42:45,514 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7726aa0d54b94a279db0fb6a24a00598, entries=150, sequenceid=170, filesize=30.4 K 2024-12-05T07:42:45,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/c9ee93d0691943fa9c368d3c2614e684 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/c9ee93d0691943fa9c368d3c2614e684 2024-12-05T07:42:45,518 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/c9ee93d0691943fa9c368d3c2614e684, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T07:42:45,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/239bdf255c2c46a0a12869fbee6ba037 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/239bdf255c2c46a0a12869fbee6ba037 2024-12-05T07:42:45,521 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/239bdf255c2c46a0a12869fbee6ba037, entries=150, sequenceid=170, filesize=11.9 K 2024-12-05T07:42:45,521 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e2b2ca43411f4bcc833039c57163fe52 in 1251ms, sequenceid=170, compaction requested=false 2024-12-05T07:42:45,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:45,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:45,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-05T07:42:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-05T07:42:45,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-05T07:42:45,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6250 sec 2024-12-05T07:42:45,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.6290 sec 2024-12-05T07:42:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:45,738 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:45,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-12-05T07:42:45,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384625748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384625749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384625749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ed0e7ab0f0f24a0caf04914fd45eaad6_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384565111/Put/seqid=0 2024-12-05T07:42:45,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742365_1541 (size=17284) 2024-12-05T07:42:45,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384625856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384625856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384625856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:45,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:46,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384626066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:46,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384626066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:46,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384626066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:46,190 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:46,192 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ed0e7ab0f0f24a0caf04914fd45eaad6_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ed0e7ab0f0f24a0caf04914fd45eaad6_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:46,193 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/898f48ed0ebe4058a0538305a24564e0, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:46,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/898f48ed0ebe4058a0538305a24564e0 is 175, key is test_row_0/A:col10/1733384565111/Put/seqid=0 2024-12-05T07:42:46,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742366_1542 (size=48389) 2024-12-05T07:42:46,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384626376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:46,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384626376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:46,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384626376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:46,596 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/898f48ed0ebe4058a0538305a24564e0 2024-12-05T07:42:46,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/6ce01c3ed6b64811b5995cbff778f881 is 50, key is test_row_0/B:col10/1733384565111/Put/seqid=0 2024-12-05T07:42:46,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742367_1543 (size=12151) 2024-12-05T07:42:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384626885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384626885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:46,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384626886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:47,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/6ce01c3ed6b64811b5995cbff778f881 2024-12-05T07:42:47,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/8270ac750e084b229b74ff531ba4aa26 is 50, key is test_row_0/C:col10/1733384565111/Put/seqid=0 2024-12-05T07:42:47,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742368_1544 (size=12151) 2024-12-05T07:42:47,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/8270ac750e084b229b74ff531ba4aa26 2024-12-05T07:42:47,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/898f48ed0ebe4058a0538305a24564e0 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/898f48ed0ebe4058a0538305a24564e0 2024-12-05T07:42:47,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/898f48ed0ebe4058a0538305a24564e0, entries=250, sequenceid=198, filesize=47.3 K 2024-12-05T07:42:47,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/6ce01c3ed6b64811b5995cbff778f881 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6ce01c3ed6b64811b5995cbff778f881 2024-12-05T07:42:47,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6ce01c3ed6b64811b5995cbff778f881, entries=150, sequenceid=198, filesize=11.9 K 2024-12-05T07:42:47,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/8270ac750e084b229b74ff531ba4aa26 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8270ac750e084b229b74ff531ba4aa26 2024-12-05T07:42:47,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8270ac750e084b229b74ff531ba4aa26, entries=150, sequenceid=198, filesize=11.9 K 2024-12-05T07:42:47,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e2b2ca43411f4bcc833039c57163fe52 in 1287ms, sequenceid=198, compaction requested=true 2024-12-05T07:42:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:47,025 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:47,025 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:47,026 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:47,026 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:47,026 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/A is initiating minor compaction (all files) 2024-12-05T07:42:47,026 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/B is initiating minor compaction (all files) 2024-12-05T07:42:47,026 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/A in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:47,026 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/B in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:47,026 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/d51b55f08c3347a6aa810f8261473b59, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/c9ee93d0691943fa9c368d3c2614e684, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6ce01c3ed6b64811b5995cbff778f881] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=35.9 K 2024-12-05T07:42:47,026 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/fcb1a94f8fe847bda8045d6817a658e3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7726aa0d54b94a279db0fb6a24a00598, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/898f48ed0ebe4058a0538305a24564e0] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=108.3 K 2024-12-05T07:42:47,027 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:47,027 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/fcb1a94f8fe847bda8045d6817a658e3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7726aa0d54b94a279db0fb6a24a00598, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/898f48ed0ebe4058a0538305a24564e0] 2024-12-05T07:42:47,027 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting d51b55f08c3347a6aa810f8261473b59, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733384562290 2024-12-05T07:42:47,027 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcb1a94f8fe847bda8045d6817a658e3, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733384562290 2024-12-05T07:42:47,027 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ee93d0691943fa9c368d3c2614e684, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733384562930 2024-12-05T07:42:47,027 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7726aa0d54b94a279db0fb6a24a00598, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733384562930 2024-12-05T07:42:47,027 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce01c3ed6b64811b5995cbff778f881, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384565107 2024-12-05T07:42:47,027 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 898f48ed0ebe4058a0538305a24564e0, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384565103 2024-12-05T07:42:47,032 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:47,033 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:47,033 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412052b68a49af74548b7b4f8f37080f1442b_e2b2ca43411f4bcc833039c57163fe52 store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:47,033 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/ae52abb646fe430c91fdc9fe405cc277 is 50, key is test_row_0/B:col10/1733384565111/Put/seqid=0 2024-12-05T07:42:47,035 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412052b68a49af74548b7b4f8f37080f1442b_e2b2ca43411f4bcc833039c57163fe52, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:47,035 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052b68a49af74548b7b4f8f37080f1442b_e2b2ca43411f4bcc833039c57163fe52 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:47,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742370_1546 (size=4469) 2024-12-05T07:42:47,039 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#A#compaction#458 average throughput is 4.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:47,039 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/d0a79991c2624f7e9da7ce5252a00a18 is 175, key is test_row_0/A:col10/1733384565111/Put/seqid=0 2024-12-05T07:42:47,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742369_1545 (size=12595) 2024-12-05T07:42:47,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742371_1547 (size=31549) 2024-12-05T07:42:47,053 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/ae52abb646fe430c91fdc9fe405cc277 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/ae52abb646fe430c91fdc9fe405cc277 2024-12-05T07:42:47,057 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/B of e2b2ca43411f4bcc833039c57163fe52 into ae52abb646fe430c91fdc9fe405cc277(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:47,057 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:47,057 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/B, priority=13, startTime=1733384567025; duration=0sec 2024-12-05T07:42:47,057 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:47,057 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:B 2024-12-05T07:42:47,057 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:47,058 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:47,058 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/C is initiating minor compaction (all files) 2024-12-05T07:42:47,058 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/C in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:47,058 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/d4c3588c5b0647dc810ff088e238aeb3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/239bdf255c2c46a0a12869fbee6ba037, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8270ac750e084b229b74ff531ba4aa26] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=35.9 K 2024-12-05T07:42:47,059 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting d4c3588c5b0647dc810ff088e238aeb3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733384562290 2024-12-05T07:42:47,059 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 239bdf255c2c46a0a12869fbee6ba037, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733384562930 2024-12-05T07:42:47,059 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 8270ac750e084b229b74ff531ba4aa26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384565107 2024-12-05T07:42:47,064 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#C#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:47,064 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/505aa5c9913a4317b8f06acf3e693c1d is 50, key is test_row_0/C:col10/1733384565111/Put/seqid=0 2024-12-05T07:42:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T07:42:47,066 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:47,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:47,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742372_1548 (size=12595) 2024-12-05T07:42:47,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-05T07:42:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T07:42:47,069 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:47,070 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:47,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:47,073 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/505aa5c9913a4317b8f06acf3e693c1d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/505aa5c9913a4317b8f06acf3e693c1d 2024-12-05T07:42:47,077 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/C of e2b2ca43411f4bcc833039c57163fe52 into 505aa5c9913a4317b8f06acf3e693c1d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:47,077 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:47,077 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/C, priority=13, startTime=1733384567025; duration=0sec 2024-12-05T07:42:47,077 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:47,077 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:C 2024-12-05T07:42:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T07:42:47,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T07:42:47,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:47,224 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-12-05T07:42:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a22af6599a2c46049ce62ebb51deea47_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384565744/Put/seqid=0 2024-12-05T07:42:47,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742373_1549 (size=12304) 2024-12-05T07:42:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T07:42:47,458 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/d0a79991c2624f7e9da7ce5252a00a18 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d0a79991c2624f7e9da7ce5252a00a18 2024-12-05T07:42:47,462 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/A of e2b2ca43411f4bcc833039c57163fe52 into d0a79991c2624f7e9da7ce5252a00a18(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:47,462 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:47,462 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/A, priority=13, startTime=1733384567025; duration=0sec 2024-12-05T07:42:47,462 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:47,462 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:A 2024-12-05T07:42:47,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:47,637 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a22af6599a2c46049ce62ebb51deea47_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a22af6599a2c46049ce62ebb51deea47_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:47,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/e896176d6a2144669ab2636860bc5cf1, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:47,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/e896176d6a2144669ab2636860bc5cf1 is 175, key is test_row_0/A:col10/1733384565744/Put/seqid=0 2024-12-05T07:42:47,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742374_1550 (size=31105) 2024-12-05T07:42:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T07:42:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:47,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:47,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:47,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384627924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384627924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:47,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384627925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384628035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384628036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384628036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:48,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:48,041 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/e896176d6a2144669ab2636860bc5cf1 2024-12-05T07:42:48,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e9f116f892ef4f259a98c1d3a1f93d0d is 50, key is test_row_0/B:col10/1733384565744/Put/seqid=0 2024-12-05T07:42:48,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742375_1551 (size=12151) 2024-12-05T07:42:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T07:42:48,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384628245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384628245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384628245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:48,450 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e9f116f892ef4f259a98c1d3a1f93d0d 2024-12-05T07:42:48,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/e557a338eaad4c7086c25ab43e8ab9f9 is 50, key is test_row_0/C:col10/1733384565744/Put/seqid=0 2024-12-05T07:42:48,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742376_1552 (size=12151) 2024-12-05T07:42:48,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384628556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:48,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:48,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384628556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384628556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:48,858 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/e557a338eaad4c7086c25ab43e8ab9f9 2024-12-05T07:42:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/e896176d6a2144669ab2636860bc5cf1 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e896176d6a2144669ab2636860bc5cf1 2024-12-05T07:42:48,864 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e896176d6a2144669ab2636860bc5cf1, entries=150, sequenceid=210, filesize=30.4 K 2024-12-05T07:42:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e9f116f892ef4f259a98c1d3a1f93d0d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e9f116f892ef4f259a98c1d3a1f93d0d 2024-12-05T07:42:48,867 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e9f116f892ef4f259a98c1d3a1f93d0d, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T07:42:48,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/e557a338eaad4c7086c25ab43e8ab9f9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/e557a338eaad4c7086c25ab43e8ab9f9 2024-12-05T07:42:48,870 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/e557a338eaad4c7086c25ab43e8ab9f9, entries=150, sequenceid=210, filesize=11.9 K 2024-12-05T07:42:48,870 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e2b2ca43411f4bcc833039c57163fe52 in 1646ms, sequenceid=210, compaction requested=false 2024-12-05T07:42:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-05T07:42:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-05T07:42:48,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-05T07:42:48,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-12-05T07:42:48,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8060 sec 2024-12-05T07:42:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:49,068 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:49,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-12-05T07:42:49,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054de5350391dc4305b1cc9148aa6fe34e_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384569068/Put/seqid=0 2024-12-05T07:42:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384629074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384629077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384629077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742377_1553 (size=14794) 2024-12-05T07:42:49,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384629186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384629186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384629186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T07:42:49,215 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:49,216 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-05T07:42:49,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-05T07:42:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T07:42:49,217 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T07:42:49,218 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T07:42:49,218 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T07:42:49,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T07:42:49,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T07:42:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:49,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384629396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384629396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384629396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,481 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:49,484 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054de5350391dc4305b1cc9148aa6fe34e_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054de5350391dc4305b1cc9148aa6fe34e_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:49,485 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/11bf9435478c4b569834b4010bd70641, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:49,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/11bf9435478c4b569834b4010bd70641 is 175, key is test_row_0/A:col10/1733384569068/Put/seqid=0 2024-12-05T07:42:49,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742378_1554 (size=39749) 2024-12-05T07:42:49,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T07:42:49,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:49,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:49,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:49,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T07:42:49,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T07:42:49,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:49,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:49,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:49,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384629705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384629705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384629705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:49,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T07:42:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:49,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T07:42:49,889 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/11bf9435478c4b569834b4010bd70641 2024-12-05T07:42:49,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/06c1d10f5de740efbbad5a4857070e41 is 50, key is test_row_0/B:col10/1733384569068/Put/seqid=0 2024-12-05T07:42:49,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742379_1555 (size=12151) 2024-12-05T07:42:49,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/06c1d10f5de740efbbad5a4857070e41 2024-12-05T07:42:49,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/5dc77d9babd346529ab9e0db4b523e45 is 50, key is test_row_0/C:col10/1733384569068/Put/seqid=0 2024-12-05T07:42:49,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742380_1556 (size=12151) 2024-12-05T07:42:49,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T07:42:49,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:49,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:49,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:49,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:50,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T07:42:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:50,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:50,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:50,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384630215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:50,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:50,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384630215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:50,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384630215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:50,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T07:42:50,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:50,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:50,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:50,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:50,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:50,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T07:42:50,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/5dc77d9babd346529ab9e0db4b523e45 2024-12-05T07:42:50,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/11bf9435478c4b569834b4010bd70641 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/11bf9435478c4b569834b4010bd70641 2024-12-05T07:42:50,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/11bf9435478c4b569834b4010bd70641, entries=200, sequenceid=240, filesize=38.8 K 2024-12-05T07:42:50,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/06c1d10f5de740efbbad5a4857070e41 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/06c1d10f5de740efbbad5a4857070e41 2024-12-05T07:42:50,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/06c1d10f5de740efbbad5a4857070e41, entries=150, sequenceid=240, filesize=11.9 K 2024-12-05T07:42:50,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/5dc77d9babd346529ab9e0db4b523e45 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/5dc77d9babd346529ab9e0db4b523e45 2024-12-05T07:42:50,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/5dc77d9babd346529ab9e0db4b523e45, entries=150, sequenceid=240, filesize=11.9 K 2024-12-05T07:42:50,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for e2b2ca43411f4bcc833039c57163fe52 in 1248ms, sequenceid=240, compaction requested=true 2024-12-05T07:42:50,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:50,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:50,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:50,317 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:50,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:50,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:50,317 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:50,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:50,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:50,319 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:50,319 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:50,319 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/A is initiating minor compaction (all files) 2024-12-05T07:42:50,319 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/B is initiating minor compaction (all files) 2024-12-05T07:42:50,319 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/A in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:50,319 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/B in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:50,319 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d0a79991c2624f7e9da7ce5252a00a18, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e896176d6a2144669ab2636860bc5cf1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/11bf9435478c4b569834b4010bd70641] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=100.0 K 2024-12-05T07:42:50,319 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/ae52abb646fe430c91fdc9fe405cc277, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e9f116f892ef4f259a98c1d3a1f93d0d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/06c1d10f5de740efbbad5a4857070e41] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=36.0 K 2024-12-05T07:42:50,319 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:50,319 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d0a79991c2624f7e9da7ce5252a00a18, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e896176d6a2144669ab2636860bc5cf1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/11bf9435478c4b569834b4010bd70641] 2024-12-05T07:42:50,320 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting ae52abb646fe430c91fdc9fe405cc277, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384565107 2024-12-05T07:42:50,320 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0a79991c2624f7e9da7ce5252a00a18, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384565107 2024-12-05T07:42:50,320 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting e9f116f892ef4f259a98c1d3a1f93d0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733384565744 2024-12-05T07:42:50,320 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting e896176d6a2144669ab2636860bc5cf1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733384565744 2024-12-05T07:42:50,320 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11bf9435478c4b569834b4010bd70641, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733384567920 2024-12-05T07:42:50,320 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 06c1d10f5de740efbbad5a4857070e41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733384567920 2024-12-05T07:42:50,343 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:50,344 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#B#compaction#467 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:50,344 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/d4944193a92e4b7d95d4e8a224a51026 is 50, key is test_row_0/B:col10/1733384569068/Put/seqid=0 2024-12-05T07:42:50,345 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205dcb8bb9a66e44ecd828ed86e596503d6_e2b2ca43411f4bcc833039c57163fe52 store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:50,347 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205dcb8bb9a66e44ecd828ed86e596503d6_e2b2ca43411f4bcc833039c57163fe52, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:50,347 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205dcb8bb9a66e44ecd828ed86e596503d6_e2b2ca43411f4bcc833039c57163fe52 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:50,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742381_1557 (size=4469) 2024-12-05T07:42:50,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T07:42:50,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742382_1558 (size=12697) 2024-12-05T07:42:50,361 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/d4944193a92e4b7d95d4e8a224a51026 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/d4944193a92e4b7d95d4e8a224a51026 2024-12-05T07:42:50,365 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/B of e2b2ca43411f4bcc833039c57163fe52 into d4944193a92e4b7d95d4e8a224a51026(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:50,365 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:50,365 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/B, priority=13, startTime=1733384570317; duration=0sec 2024-12-05T07:42:50,365 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:50,365 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:B 2024-12-05T07:42:50,365 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:50,366 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:50,366 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/C is initiating minor compaction (all files) 2024-12-05T07:42:50,366 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/C in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:50,366 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/505aa5c9913a4317b8f06acf3e693c1d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/e557a338eaad4c7086c25ab43e8ab9f9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/5dc77d9babd346529ab9e0db4b523e45] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=36.0 K 2024-12-05T07:42:50,367 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 505aa5c9913a4317b8f06acf3e693c1d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1733384565107 2024-12-05T07:42:50,367 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting e557a338eaad4c7086c25ab43e8ab9f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733384565744 2024-12-05T07:42:50,367 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dc77d9babd346529ab9e0db4b523e45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733384567920 2024-12-05T07:42:50,373 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:50,373 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/0c3f27c458f74b91bb6940cba0879b0b is 50, key is test_row_0/C:col10/1733384569068/Put/seqid=0 2024-12-05T07:42:50,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742383_1559 (size=12697) 2024-12-05T07:42:50,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36693 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T07:42:50,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:50,438 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-12-05T07:42:50,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051d5533b2051f41ca930a3e3d0ce2bf02_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384569072/Put/seqid=0 2024-12-05T07:42:50,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742384_1560 (size=12304) 2024-12-05T07:42:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:50,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. as already flushing 2024-12-05T07:42:50,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384630649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:50,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384630649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:50,752 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#A#compaction#468 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:50,752 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/dca8236053864f1c8a8fb4d9c05d3bd9 is 175, key is test_row_0/A:col10/1733384569068/Put/seqid=0 2024-12-05T07:42:50,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384630756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:50,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384630756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:50,783 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/0c3f27c458f74b91bb6940cba0879b0b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0c3f27c458f74b91bb6940cba0879b0b 2024-12-05T07:42:50,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742385_1561 (size=31651) 2024-12-05T07:42:50,788 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/C of e2b2ca43411f4bcc833039c57163fe52 into 0c3f27c458f74b91bb6940cba0879b0b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:50,788 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:50,788 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/C, priority=13, startTime=1733384570317; duration=0sec 2024-12-05T07:42:50,788 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:50,789 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:C 2024-12-05T07:42:50,791 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/dca8236053864f1c8a8fb4d9c05d3bd9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/dca8236053864f1c8a8fb4d9c05d3bd9 2024-12-05T07:42:50,794 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/A of e2b2ca43411f4bcc833039c57163fe52 into dca8236053864f1c8a8fb4d9c05d3bd9(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:50,794 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:50,794 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/A, priority=13, startTime=1733384570316; duration=0sec 2024-12-05T07:42:50,795 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:50,795 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:A 2024-12-05T07:42:50,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:50,848 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412051d5533b2051f41ca930a3e3d0ce2bf02_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051d5533b2051f41ca930a3e3d0ce2bf02_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:50,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7ea81e1c677949e487e91449e6bd0619, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:50,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7ea81e1c677949e487e91449e6bd0619 is 175, key is test_row_0/A:col10/1733384569072/Put/seqid=0 2024-12-05T07:42:50,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742386_1562 (size=31105) 2024-12-05T07:42:50,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384630965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:50,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384630967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:50,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:50,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:51,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37804 deadline: 1733384631225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:51,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37818 deadline: 1733384631225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:51,228 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in e2b2ca43411f4bcc833039c57163fe52 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:51,228 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in e2b2ca43411f4bcc833039c57163fe52 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:51,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384631236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:51,237 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in e2b2ca43411f4bcc833039c57163fe52 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:51,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:51,253 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7ea81e1c677949e487e91449e6bd0619 2024-12-05T07:42:51,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/dfe2735dcb0d496097261469bb7d11b8 is 50, key is test_row_0/B:col10/1733384569072/Put/seqid=0 2024-12-05T07:42:51,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742387_1563 (size=12151) 2024-12-05T07:42:51,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384631276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:51,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384631276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,277 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T07:42:51,670 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/dfe2735dcb0d496097261469bb7d11b8 2024-12-05T07:42:51,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/6a113d013154456bb4452d3ba86f3707 is 50, key is test_row_0/C:col10/1733384569072/Put/seqid=0 2024-12-05T07:42:51,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742388_1564 (size=12151) 2024-12-05T07:42:51,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37786 deadline: 1733384631786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:51,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37824 deadline: 1733384631786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:52,078 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/6a113d013154456bb4452d3ba86f3707 2024-12-05T07:42:52,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/7ea81e1c677949e487e91449e6bd0619 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7ea81e1c677949e487e91449e6bd0619 2024-12-05T07:42:52,084 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7ea81e1c677949e487e91449e6bd0619, entries=150, sequenceid=249, filesize=30.4 K 2024-12-05T07:42:52,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/dfe2735dcb0d496097261469bb7d11b8 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/dfe2735dcb0d496097261469bb7d11b8 2024-12-05T07:42:52,087 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/dfe2735dcb0d496097261469bb7d11b8, entries=150, sequenceid=249, filesize=11.9 K 2024-12-05T07:42:52,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/6a113d013154456bb4452d3ba86f3707 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/6a113d013154456bb4452d3ba86f3707 2024-12-05T07:42:52,090 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/6a113d013154456bb4452d3ba86f3707, entries=150, sequenceid=249, filesize=11.9 K 2024-12-05T07:42:52,091 INFO [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for e2b2ca43411f4bcc833039c57163fe52 in 1653ms, sequenceid=249, compaction requested=false 2024-12-05T07:42:52,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:52,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:52,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fc6cd38557f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-05T07:42:52,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-05T07:42:52,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-05T07:42:52,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8740 sec 2024-12-05T07:42:52,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.8780 sec 2024-12-05T07:42:52,274 INFO [Thread-2232 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2232. 2024-12-05T07:42:52,274 DEBUG [Thread-2232 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:52,274 DEBUG [Thread-2232 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,274 DEBUG [Thread-2232 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:52,275 INFO [Thread-2230 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2230. 2024-12-05T07:42:52,275 DEBUG [Thread-2230 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:52,275 DEBUG [Thread-2230 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,275 DEBUG [Thread-2230 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:52,275 INFO [Thread-2234 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2234. 2024-12-05T07:42:52,276 DEBUG [Thread-2234 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:52,276 DEBUG [Thread-2234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,276 DEBUG [Thread-2234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:52,276 INFO [Thread-2236 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2236. 2024-12-05T07:42:52,276 DEBUG [Thread-2236 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:52,276 DEBUG [Thread-2236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,276 DEBUG [Thread-2236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:52,276 INFO [Thread-2228 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2228. 2024-12-05T07:42:52,276 DEBUG [Thread-2228 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:52,277 DEBUG [Thread-2228 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,277 DEBUG [Thread-2228 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:52,575 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T07:42:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(8855): Flush requested on e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:52,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-05T07:42:52,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-12-05T07:42:52,801 INFO [Thread-2221 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2221. 2024-12-05T07:42:52,801 DEBUG [Thread-2221 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:52,801 DEBUG [Thread-2221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,801 DEBUG [Thread-2221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:52,809 INFO [Thread-2225 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2225. 2024-12-05T07:42:52,809 DEBUG [Thread-2225 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:52,809 DEBUG [Thread-2225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,809 DEBUG [Thread-2225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:52,809 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:52,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ef12ca26446a4c4d82a53887c3c5888c_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_0/A:col10/1733384570647/Put/seqid=0 2024-12-05T07:42:52,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742389_1565 (size=12454) 2024-12-05T07:42:53,215 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:53,223 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ef12ca26446a4c4d82a53887c3c5888c_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ef12ca26446a4c4d82a53887c3c5888c_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:53,225 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/d7dd382429104a828e8d272a898866f5, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:53,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/d7dd382429104a828e8d272a898866f5 is 175, key is test_row_0/A:col10/1733384570647/Put/seqid=0 2024-12-05T07:42:53,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742390_1566 (size=31255) 2024-12-05T07:42:53,237 INFO [Thread-2217 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2217. 2024-12-05T07:42:53,237 DEBUG [Thread-2217 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:53,237 DEBUG [Thread-2217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:53,237 DEBUG [Thread-2217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:53,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:53,248 INFO [Thread-2219 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2219. 2024-12-05T07:42:53,248 DEBUG [Thread-2219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:53,248 DEBUG [Thread-2219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:53,248 DEBUG [Thread-2219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:53,248 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:53,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T07:42:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36693 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37778 deadline: 1733384633267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:53,268 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fc6cd38557f3,36693,1733384394401 for 'test_row_2' in e2b2ca43411f4bcc833039c57163fe52 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-05T07:42:53,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:53,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T07:42:53,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., hostname=fc6cd38557f3,36693,1733384394401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T07:42:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T07:42:53,387 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:53,631 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/d7dd382429104a828e8d272a898866f5 2024-12-05T07:42:53,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/771f216cd0bc4e17a5fd8ab1b6139743 is 50, key is test_row_0/B:col10/1733384570647/Put/seqid=0 2024-12-05T07:42:53,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742391_1567 (size=12301) 2024-12-05T07:42:54,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/771f216cd0bc4e17a5fd8ab1b6139743 2024-12-05T07:42:54,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/0782243bf9294a27a9b21f11258f2d9b is 50, key is test_row_0/C:col10/1733384570647/Put/seqid=0 2024-12-05T07:42:54,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742392_1568 (size=12301) 2024-12-05T07:42:54,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/0782243bf9294a27a9b21f11258f2d9b 2024-12-05T07:42:54,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/d7dd382429104a828e8d272a898866f5 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d7dd382429104a828e8d272a898866f5 2024-12-05T07:42:54,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d7dd382429104a828e8d272a898866f5, entries=150, sequenceid=281, filesize=30.5 K 2024-12-05T07:42:54,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/771f216cd0bc4e17a5fd8ab1b6139743 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/771f216cd0bc4e17a5fd8ab1b6139743 2024-12-05T07:42:54,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/771f216cd0bc4e17a5fd8ab1b6139743, entries=150, sequenceid=281, filesize=12.0 K 2024-12-05T07:42:54,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/0782243bf9294a27a9b21f11258f2d9b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0782243bf9294a27a9b21f11258f2d9b 2024-12-05T07:42:54,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0782243bf9294a27a9b21f11258f2d9b, entries=150, sequenceid=281, filesize=12.0 K 2024-12-05T07:42:54,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for e2b2ca43411f4bcc833039c57163fe52 in 1689ms, sequenceid=281, compaction requested=true 2024-12-05T07:42:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:54,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T07:42:54,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:54,490 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:54,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T07:42:54,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:54,490 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:54,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e2b2ca43411f4bcc833039c57163fe52:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T07:42:54,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:54,490 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:54,490 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:54,490 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/A is initiating minor compaction (all files) 2024-12-05T07:42:54,490 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/B is initiating minor compaction (all files) 2024-12-05T07:42:54,490 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/A in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:54,490 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/B in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:54,491 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/d4944193a92e4b7d95d4e8a224a51026, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/dfe2735dcb0d496097261469bb7d11b8, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/771f216cd0bc4e17a5fd8ab1b6139743] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=36.3 K 2024-12-05T07:42:54,491 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/dca8236053864f1c8a8fb4d9c05d3bd9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7ea81e1c677949e487e91449e6bd0619, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d7dd382429104a828e8d272a898866f5] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=91.8 K 2024-12-05T07:42:54,491 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:54,491 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. files: [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/dca8236053864f1c8a8fb4d9c05d3bd9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7ea81e1c677949e487e91449e6bd0619, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d7dd382429104a828e8d272a898866f5] 2024-12-05T07:42:54,491 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting d4944193a92e4b7d95d4e8a224a51026, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733384567920 2024-12-05T07:42:54,491 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca8236053864f1c8a8fb4d9c05d3bd9, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733384567920 2024-12-05T07:42:54,491 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe2735dcb0d496097261469bb7d11b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733384569071 2024-12-05T07:42:54,491 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ea81e1c677949e487e91449e6bd0619, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733384569071 2024-12-05T07:42:54,491 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 771f216cd0bc4e17a5fd8ab1b6139743, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733384570643 2024-12-05T07:42:54,491 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7dd382429104a828e8d272a898866f5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733384570643 2024-12-05T07:42:54,495 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:54,495 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#B#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:54,496 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/f6cd98a45b6c4724a02e3979301cdcc9 is 50, key is test_row_0/B:col10/1733384570647/Put/seqid=0 2024-12-05T07:42:54,496 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120595dab9e4d8fc4fc89b286473b72313ba_e2b2ca43411f4bcc833039c57163fe52 store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:54,497 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120595dab9e4d8fc4fc89b286473b72313ba_e2b2ca43411f4bcc833039c57163fe52, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:54,497 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120595dab9e4d8fc4fc89b286473b72313ba_e2b2ca43411f4bcc833039c57163fe52 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:54,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742393_1569 (size=12949) 2024-12-05T07:42:54,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742394_1570 (size=4469) 2024-12-05T07:42:54,903 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#A#compaction#477 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:54,905 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/4458c6e0feb8421fb24c2eed2febee9d is 175, key is test_row_0/A:col10/1733384570647/Put/seqid=0 2024-12-05T07:42:54,910 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/f6cd98a45b6c4724a02e3979301cdcc9 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/f6cd98a45b6c4724a02e3979301cdcc9 2024-12-05T07:42:54,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742395_1571 (size=31903) 2024-12-05T07:42:54,914 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/B of e2b2ca43411f4bcc833039c57163fe52 into f6cd98a45b6c4724a02e3979301cdcc9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:54,914 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:54,914 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/4458c6e0feb8421fb24c2eed2febee9d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/4458c6e0feb8421fb24c2eed2febee9d 2024-12-05T07:42:54,914 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/B, priority=13, startTime=1733384574490; duration=0sec 2024-12-05T07:42:54,914 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T07:42:54,914 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:B 2024-12-05T07:42:54,915 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T07:42:54,915 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T07:42:54,915 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1541): e2b2ca43411f4bcc833039c57163fe52/C is initiating minor compaction (all files) 2024-12-05T07:42:54,915 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2b2ca43411f4bcc833039c57163fe52/C in TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:54,915 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0c3f27c458f74b91bb6940cba0879b0b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/6a113d013154456bb4452d3ba86f3707, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0782243bf9294a27a9b21f11258f2d9b] into tmpdir=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp, totalSize=36.3 K 2024-12-05T07:42:54,916 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c3f27c458f74b91bb6940cba0879b0b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1733384567920 2024-12-05T07:42:54,916 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a113d013154456bb4452d3ba86f3707, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733384569071 2024-12-05T07:42:54,916 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] compactions.Compactor(225): Compacting 0782243bf9294a27a9b21f11258f2d9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733384570643 2024-12-05T07:42:54,918 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/A of e2b2ca43411f4bcc833039c57163fe52 into 4458c6e0feb8421fb24c2eed2febee9d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:54,918 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:54,918 INFO [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/A, priority=13, startTime=1733384574489; duration=0sec 2024-12-05T07:42:54,918 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:54,918 DEBUG [RS:0;fc6cd38557f3:36693-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:A 2024-12-05T07:42:54,922 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2b2ca43411f4bcc833039c57163fe52#C#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T07:42:54,922 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/404cd4cefd504a59a8866268cce34d6b is 50, key is test_row_0/C:col10/1733384570647/Put/seqid=0 2024-12-05T07:42:54,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742396_1572 (size=12949) 2024-12-05T07:42:55,334 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/404cd4cefd504a59a8866268cce34d6b as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/404cd4cefd504a59a8866268cce34d6b 2024-12-05T07:42:55,341 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e2b2ca43411f4bcc833039c57163fe52/C of e2b2ca43411f4bcc833039c57163fe52 into 404cd4cefd504a59a8866268cce34d6b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T07:42:55,341 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2b2ca43411f4bcc833039c57163fe52: 2024-12-05T07:42:55,341 INFO [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., storeName=e2b2ca43411f4bcc833039c57163fe52/C, priority=13, startTime=1733384574490; duration=0sec 2024-12-05T07:42:55,341 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T07:42:55,341 DEBUG [RS:0;fc6cd38557f3:36693-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2b2ca43411f4bcc833039c57163fe52:C 2024-12-05T07:42:57,301 INFO [Thread-2223 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2223. 2024-12-05T07:42:57,301 DEBUG [Thread-2223 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T07:42:57,301 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:57,302 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:57,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6025 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6056 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5868 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5996 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6046 2024-12-05T07:42:57,303 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T07:42:57,303 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T07:42:57,304 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T07:42:57,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:57,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:57,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:57,306 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-05T07:42:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T07:42:57,310 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384577309"}]},"ts":"1733384577309"} 2024-12-05T07:42:57,312 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T07:42:57,312 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T07:42:57,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T07:42:57,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, UNASSIGN}] 2024-12-05T07:42:57,315 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, UNASSIGN 2024-12-05T07:42:57,316 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=e2b2ca43411f4bcc833039c57163fe52, regionState=CLOSING, regionLocation=fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:57,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, UNASSIGN because future has completed 2024-12-05T07:42:57,319 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T07:42:57,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401}] 2024-12-05T07:42:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T07:42:57,473 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(122): Close e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:57,473 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T07:42:57,473 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1722): Closing e2b2ca43411f4bcc833039c57163fe52, disabling compactions & flushes 2024-12-05T07:42:57,473 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:57,473 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:57,474 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. after waiting 0 ms 2024-12-05T07:42:57,474 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:57,474 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(2902): Flushing e2b2ca43411f4bcc833039c57163fe52 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-12-05T07:42:57,484 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053e10976e9ca94dd7ab1127efc2391315_e2b2ca43411f4bcc833039c57163fe52 is 50, key is test_row_1/A:col10/1733384572807/Put/seqid=0 2024-12-05T07:42:57,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742397_1573 (size=9914) 2024-12-05T07:42:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T07:42:57,888 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T07:42:57,894 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053e10976e9ca94dd7ab1127efc2391315_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053e10976e9ca94dd7ab1127efc2391315_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:57,896 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/20230a29974c4a36acf609b7fb6b953c, store: [table=TestAcidGuarantees family=A region=e2b2ca43411f4bcc833039c57163fe52] 2024-12-05T07:42:57,896 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/20230a29974c4a36acf609b7fb6b953c is 175, key is test_row_1/A:col10/1733384572807/Put/seqid=0 2024-12-05T07:42:57,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742398_1574 (size=22561) 2024-12-05T07:42:57,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T07:42:58,302 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/20230a29974c4a36acf609b7fb6b953c 2024-12-05T07:42:58,314 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e5128b85e4654b4d9bd616d2b632da4c is 50, key is test_row_1/B:col10/1733384572807/Put/seqid=0 2024-12-05T07:42:58,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742399_1575 (size=9857) 2024-12-05T07:42:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T07:42:58,718 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e5128b85e4654b4d9bd616d2b632da4c 2024-12-05T07:42:58,726 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/0823104db53a4d799ca316db4457ac66 is 50, key is test_row_1/C:col10/1733384572807/Put/seqid=0 2024-12-05T07:42:58,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742400_1576 (size=9857) 2024-12-05T07:42:59,131 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/0823104db53a4d799ca316db4457ac66 2024-12-05T07:42:59,139 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/A/20230a29974c4a36acf609b7fb6b953c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/20230a29974c4a36acf609b7fb6b953c 2024-12-05T07:42:59,142 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/20230a29974c4a36acf609b7fb6b953c, entries=100, sequenceid=291, filesize=22.0 K 2024-12-05T07:42:59,143 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/B/e5128b85e4654b4d9bd616d2b632da4c as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e5128b85e4654b4d9bd616d2b632da4c 2024-12-05T07:42:59,145 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e5128b85e4654b4d9bd616d2b632da4c, entries=100, sequenceid=291, filesize=9.6 K 2024-12-05T07:42:59,146 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/.tmp/C/0823104db53a4d799ca316db4457ac66 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0823104db53a4d799ca316db4457ac66 2024-12-05T07:42:59,148 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0823104db53a4d799ca316db4457ac66, entries=100, sequenceid=291, filesize=9.6 K 2024-12-05T07:42:59,149 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e2b2ca43411f4bcc833039c57163fe52 in 1675ms, sequenceid=291, compaction requested=false 2024-12-05T07:42:59,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/0af9690b2dd44e2884518a800861bfd9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/1540314b437641508f8a94f78ac4ea56, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/a6717b18765043339be8e2905f09b8ed, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/3c13471d22374dffa6248e3f19648d7f, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bc66f0ebc99f4624b9bff773e23cae6b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7d9b61730160449b9f1b4cb0b1b77f57, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/519e72c2cc024bdd80f47f27929d13e1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e223280cc3fa4fc7b6b5612a4f1bb412, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bddb8d2e5bd54d8b9fd96e27c9ba2779, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/20739e55219a4ed093551f8d48dc80b1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/fcb1a94f8fe847bda8045d6817a658e3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7726aa0d54b94a279db0fb6a24a00598, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/898f48ed0ebe4058a0538305a24564e0, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d0a79991c2624f7e9da7ce5252a00a18, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e896176d6a2144669ab2636860bc5cf1, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/11bf9435478c4b569834b4010bd70641, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/dca8236053864f1c8a8fb4d9c05d3bd9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7ea81e1c677949e487e91449e6bd0619, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d7dd382429104a828e8d272a898866f5] to archive 2024-12-05T07:42:59,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:42:59,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/0af9690b2dd44e2884518a800861bfd9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/0af9690b2dd44e2884518a800861bfd9 2024-12-05T07:42:59,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/1540314b437641508f8a94f78ac4ea56 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/1540314b437641508f8a94f78ac4ea56 2024-12-05T07:42:59,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/a6717b18765043339be8e2905f09b8ed to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/a6717b18765043339be8e2905f09b8ed 2024-12-05T07:42:59,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/3c13471d22374dffa6248e3f19648d7f to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/3c13471d22374dffa6248e3f19648d7f 2024-12-05T07:42:59,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bc66f0ebc99f4624b9bff773e23cae6b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bc66f0ebc99f4624b9bff773e23cae6b 2024-12-05T07:42:59,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7d9b61730160449b9f1b4cb0b1b77f57 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7d9b61730160449b9f1b4cb0b1b77f57 2024-12-05T07:42:59,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/519e72c2cc024bdd80f47f27929d13e1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/519e72c2cc024bdd80f47f27929d13e1 2024-12-05T07:42:59,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e223280cc3fa4fc7b6b5612a4f1bb412 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e223280cc3fa4fc7b6b5612a4f1bb412 2024-12-05T07:42:59,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bddb8d2e5bd54d8b9fd96e27c9ba2779 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/bddb8d2e5bd54d8b9fd96e27c9ba2779 2024-12-05T07:42:59,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/20739e55219a4ed093551f8d48dc80b1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/20739e55219a4ed093551f8d48dc80b1 2024-12-05T07:42:59,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/fcb1a94f8fe847bda8045d6817a658e3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/fcb1a94f8fe847bda8045d6817a658e3 2024-12-05T07:42:59,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7726aa0d54b94a279db0fb6a24a00598 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7726aa0d54b94a279db0fb6a24a00598 2024-12-05T07:42:59,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/898f48ed0ebe4058a0538305a24564e0 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/898f48ed0ebe4058a0538305a24564e0 2024-12-05T07:42:59,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d0a79991c2624f7e9da7ce5252a00a18 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d0a79991c2624f7e9da7ce5252a00a18 2024-12-05T07:42:59,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e896176d6a2144669ab2636860bc5cf1 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/e896176d6a2144669ab2636860bc5cf1 2024-12-05T07:42:59,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/11bf9435478c4b569834b4010bd70641 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/11bf9435478c4b569834b4010bd70641 2024-12-05T07:42:59,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/dca8236053864f1c8a8fb4d9c05d3bd9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/dca8236053864f1c8a8fb4d9c05d3bd9 2024-12-05T07:42:59,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7ea81e1c677949e487e91449e6bd0619 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/7ea81e1c677949e487e91449e6bd0619 2024-12-05T07:42:59,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d7dd382429104a828e8d272a898866f5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/d7dd382429104a828e8d272a898866f5 2024-12-05T07:42:59,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/5001887f0cd9431e8777d67bf9b58d1d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/410ece1bdca048f899bc5d4c6ba9ea09, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/a1a6f1100a9c4cf5ac1f28afc1f64c61, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/b74413cc31ea4779b76b1b405e1f57ef, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e2872981610c454fafd1644d673a4786, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6f1bc2b066194b3082496e6c670d3151, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/f10a6930c5c443c1b5b32c7189c0a684, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/ae35f3fdd875482ab23c60fa800a7ceb, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e7be8d26f86d4b2ca8ca2ffe1e97ce33, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/d51b55f08c3347a6aa810f8261473b59, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/cf464fee7024413788a9be56c1df3fae, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/c9ee93d0691943fa9c368d3c2614e684, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/ae52abb646fe430c91fdc9fe405cc277, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6ce01c3ed6b64811b5995cbff778f881, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e9f116f892ef4f259a98c1d3a1f93d0d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/d4944193a92e4b7d95d4e8a224a51026, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/06c1d10f5de740efbbad5a4857070e41, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/dfe2735dcb0d496097261469bb7d11b8, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/771f216cd0bc4e17a5fd8ab1b6139743] to archive 2024-12-05T07:42:59,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:42:59,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/5001887f0cd9431e8777d67bf9b58d1d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/5001887f0cd9431e8777d67bf9b58d1d 2024-12-05T07:42:59,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/410ece1bdca048f899bc5d4c6ba9ea09 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/410ece1bdca048f899bc5d4c6ba9ea09 2024-12-05T07:42:59,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/a1a6f1100a9c4cf5ac1f28afc1f64c61 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/a1a6f1100a9c4cf5ac1f28afc1f64c61 2024-12-05T07:42:59,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/b74413cc31ea4779b76b1b405e1f57ef to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/b74413cc31ea4779b76b1b405e1f57ef 2024-12-05T07:42:59,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e2872981610c454fafd1644d673a4786 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e2872981610c454fafd1644d673a4786 2024-12-05T07:42:59,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6f1bc2b066194b3082496e6c670d3151 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6f1bc2b066194b3082496e6c670d3151 2024-12-05T07:42:59,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/f10a6930c5c443c1b5b32c7189c0a684 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/f10a6930c5c443c1b5b32c7189c0a684 2024-12-05T07:42:59,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/ae35f3fdd875482ab23c60fa800a7ceb to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/ae35f3fdd875482ab23c60fa800a7ceb 2024-12-05T07:42:59,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e7be8d26f86d4b2ca8ca2ffe1e97ce33 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e7be8d26f86d4b2ca8ca2ffe1e97ce33 2024-12-05T07:42:59,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/d51b55f08c3347a6aa810f8261473b59 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/d51b55f08c3347a6aa810f8261473b59 2024-12-05T07:42:59,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/cf464fee7024413788a9be56c1df3fae to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/cf464fee7024413788a9be56c1df3fae 2024-12-05T07:42:59,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/c9ee93d0691943fa9c368d3c2614e684 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/c9ee93d0691943fa9c368d3c2614e684 2024-12-05T07:42:59,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/ae52abb646fe430c91fdc9fe405cc277 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/ae52abb646fe430c91fdc9fe405cc277 2024-12-05T07:42:59,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6ce01c3ed6b64811b5995cbff778f881 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/6ce01c3ed6b64811b5995cbff778f881 2024-12-05T07:42:59,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e9f116f892ef4f259a98c1d3a1f93d0d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e9f116f892ef4f259a98c1d3a1f93d0d 2024-12-05T07:42:59,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/d4944193a92e4b7d95d4e8a224a51026 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/d4944193a92e4b7d95d4e8a224a51026 2024-12-05T07:42:59,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/06c1d10f5de740efbbad5a4857070e41 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/06c1d10f5de740efbbad5a4857070e41 2024-12-05T07:42:59,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/dfe2735dcb0d496097261469bb7d11b8 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/dfe2735dcb0d496097261469bb7d11b8 2024-12-05T07:42:59,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/771f216cd0bc4e17a5fd8ab1b6139743 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/771f216cd0bc4e17a5fd8ab1b6139743 2024-12-05T07:42:59,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/327d9fb22d234fdab5e4a7ae09e65140, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/2ab50168e28f4eb8ac8e8cbc9b22aaf0, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/e0842fb53ffc423c9635fda16d9091ba, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/d7245618fd1942dabf5fc0f53541d8b8, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/40291d4884ca4c988b537d7b610b354b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/4524f83726a3454ab84bdef38be45dcc, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/beb8fc0f727f4d5ea593f674333298d5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8dc2ae0ca06341968316d731666375ea, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/69a447a47a9c4ceb96015414209075a5, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/d4c3588c5b0647dc810ff088e238aeb3, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/c0dc21b578ee424fabe9d71d24c3561c, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/239bdf255c2c46a0a12869fbee6ba037, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/505aa5c9913a4317b8f06acf3e693c1d, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8270ac750e084b229b74ff531ba4aa26, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/e557a338eaad4c7086c25ab43e8ab9f9, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0c3f27c458f74b91bb6940cba0879b0b, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/5dc77d9babd346529ab9e0db4b523e45, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/6a113d013154456bb4452d3ba86f3707, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0782243bf9294a27a9b21f11258f2d9b] to archive 2024-12-05T07:42:59,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T07:42:59,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/327d9fb22d234fdab5e4a7ae09e65140 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/327d9fb22d234fdab5e4a7ae09e65140 2024-12-05T07:42:59,194 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/2ab50168e28f4eb8ac8e8cbc9b22aaf0 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/2ab50168e28f4eb8ac8e8cbc9b22aaf0 2024-12-05T07:42:59,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/e0842fb53ffc423c9635fda16d9091ba to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/e0842fb53ffc423c9635fda16d9091ba 2024-12-05T07:42:59,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/d7245618fd1942dabf5fc0f53541d8b8 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/d7245618fd1942dabf5fc0f53541d8b8 2024-12-05T07:42:59,197 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/40291d4884ca4c988b537d7b610b354b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/40291d4884ca4c988b537d7b610b354b 2024-12-05T07:42:59,198 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/4524f83726a3454ab84bdef38be45dcc to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/4524f83726a3454ab84bdef38be45dcc 2024-12-05T07:42:59,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/beb8fc0f727f4d5ea593f674333298d5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/beb8fc0f727f4d5ea593f674333298d5 2024-12-05T07:42:59,199 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8dc2ae0ca06341968316d731666375ea to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8dc2ae0ca06341968316d731666375ea 2024-12-05T07:42:59,200 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/69a447a47a9c4ceb96015414209075a5 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/69a447a47a9c4ceb96015414209075a5 2024-12-05T07:42:59,201 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/d4c3588c5b0647dc810ff088e238aeb3 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/d4c3588c5b0647dc810ff088e238aeb3 2024-12-05T07:42:59,203 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/c0dc21b578ee424fabe9d71d24c3561c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/c0dc21b578ee424fabe9d71d24c3561c 2024-12-05T07:42:59,204 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/239bdf255c2c46a0a12869fbee6ba037 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/239bdf255c2c46a0a12869fbee6ba037 2024-12-05T07:42:59,205 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/505aa5c9913a4317b8f06acf3e693c1d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/505aa5c9913a4317b8f06acf3e693c1d 2024-12-05T07:42:59,206 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8270ac750e084b229b74ff531ba4aa26 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/8270ac750e084b229b74ff531ba4aa26 2024-12-05T07:42:59,207 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/e557a338eaad4c7086c25ab43e8ab9f9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/e557a338eaad4c7086c25ab43e8ab9f9 2024-12-05T07:42:59,208 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0c3f27c458f74b91bb6940cba0879b0b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0c3f27c458f74b91bb6940cba0879b0b 2024-12-05T07:42:59,210 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/5dc77d9babd346529ab9e0db4b523e45 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/5dc77d9babd346529ab9e0db4b523e45 2024-12-05T07:42:59,211 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/6a113d013154456bb4452d3ba86f3707 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/6a113d013154456bb4452d3ba86f3707 2024-12-05T07:42:59,212 DEBUG [StoreCloser-TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0782243bf9294a27a9b21f11258f2d9b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0782243bf9294a27a9b21f11258f2d9b 2024-12-05T07:42:59,217 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/recovered.edits/294.seqid, newMaxSeqId=294, maxSeqId=4 2024-12-05T07:42:59,217 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. 2024-12-05T07:42:59,217 DEBUG [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1676): Region close journal for e2b2ca43411f4bcc833039c57163fe52: Waiting for close lock at 1733384577473Running coprocessor pre-close hooks at 1733384577473Disabling compacts and flushes for region at 1733384577473Disabling writes for close at 1733384577474 (+1 ms)Obtaining lock to block concurrent updates at 1733384577474Preparing flush snapshotting stores in e2b2ca43411f4bcc833039c57163fe52 at 1733384577474Finished memstore snapshotting TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1733384577475 (+1 ms)Flushing stores of TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52. at 1733384577476 (+1 ms)Flushing e2b2ca43411f4bcc833039c57163fe52/A: creating writer at 1733384577476Flushing mob file e2b2ca43411f4bcc833039c57163fe52/A: appending metadata at 1733384577484 (+8 ms)Flushing mob file e2b2ca43411f4bcc833039c57163fe52/A: closing flushed file at 1733384577484Flushing e2b2ca43411f4bcc833039c57163fe52/A: appending metadata at 1733384577896 (+412 ms)Flushing e2b2ca43411f4bcc833039c57163fe52/A: closing flushed file at 1733384577896Flushing e2b2ca43411f4bcc833039c57163fe52/B: creating writer at 1733384578311 (+415 ms)Flushing e2b2ca43411f4bcc833039c57163fe52/B: appending metadata at 1733384578314 (+3 ms)Flushing e2b2ca43411f4bcc833039c57163fe52/B: closing flushed file at 1733384578314Flushing e2b2ca43411f4bcc833039c57163fe52/C: creating writer at 1733384578722 (+408 ms)Flushing e2b2ca43411f4bcc833039c57163fe52/C: appending metadata at 1733384578725 (+3 ms)Flushing e2b2ca43411f4bcc833039c57163fe52/C: closing flushed file at 1733384578725Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@524bd038: reopening flushed file at 1733384579138 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6af316f0: reopening flushed file at 1733384579142 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46db32e5: reopening flushed file at 1733384579145 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e2b2ca43411f4bcc833039c57163fe52 in 1675ms, sequenceid=291, compaction requested=false at 1733384579149 (+4 ms)Writing region close event to WAL at 1733384579214 (+65 ms)Running coprocessor post-close hooks at 1733384579217 (+3 ms)Closed at 1733384579217 2024-12-05T07:42:59,219 INFO [RS_CLOSE_REGION-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(157): Closed e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,219 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=e2b2ca43411f4bcc833039c57163fe52, regionState=CLOSED 2024-12-05T07:42:59,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 because future has completed 2024-12-05T07:42:59,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-05T07:42:59,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseRegionProcedure e2b2ca43411f4bcc833039c57163fe52, server=fc6cd38557f3,36693,1733384394401 in 1.9020 sec 2024-12-05T07:42:59,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-12-05T07:42:59,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e2b2ca43411f4bcc833039c57163fe52, UNASSIGN in 1.9080 sec 2024-12-05T07:42:59,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-05T07:42:59,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9120 sec 2024-12-05T07:42:59,226 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733384579225"}]},"ts":"1733384579225"} 2024-12-05T07:42:59,227 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T07:42:59,227 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T07:42:59,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9210 sec 2024-12-05T07:42:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T07:42:59,456 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:59,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-05T07:42:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:59,458 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T07:42:59,459 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=160, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:59,460 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,462 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C, FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/recovered.edits] 2024-12-05T07:42:59,464 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/20230a29974c4a36acf609b7fb6b953c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/20230a29974c4a36acf609b7fb6b953c 2024-12-05T07:42:59,465 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/4458c6e0feb8421fb24c2eed2febee9d to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/A/4458c6e0feb8421fb24c2eed2febee9d 2024-12-05T07:42:59,468 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e5128b85e4654b4d9bd616d2b632da4c to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/e5128b85e4654b4d9bd616d2b632da4c 2024-12-05T07:42:59,469 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/f6cd98a45b6c4724a02e3979301cdcc9 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/B/f6cd98a45b6c4724a02e3979301cdcc9 2024-12-05T07:42:59,470 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0823104db53a4d799ca316db4457ac66 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/0823104db53a4d799ca316db4457ac66 2024-12-05T07:42:59,470 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/404cd4cefd504a59a8866268cce34d6b to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/C/404cd4cefd504a59a8866268cce34d6b 2024-12-05T07:42:59,472 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/recovered.edits/294.seqid to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52/recovered.edits/294.seqid 2024-12-05T07:42:59,472 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/default/TestAcidGuarantees/e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,472 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T07:42:59,472 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T07:42:59,473 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T07:42:59,474 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051cc1adaf5af54abebe8a491734fe72da_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051cc1adaf5af54abebe8a491734fe72da_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,475 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051d5533b2051f41ca930a3e3d0ce2bf02_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412051d5533b2051f41ca930a3e3d0ce2bf02_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,476 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120533d790d432d14184aabd266c42a16b43_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120533d790d432d14184aabd266c42a16b43_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,476 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120535a90fd407b64b06a4bc96a116bcf8b3_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120535a90fd407b64b06a4bc96a116bcf8b3_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,477 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120537361566794f4f149a6aeb92b58e4eb6_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120537361566794f4f149a6aeb92b58e4eb6_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,477 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053e10976e9ca94dd7ab1127efc2391315_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053e10976e9ca94dd7ab1127efc2391315_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,478 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054de5350391dc4305b1cc9148aa6fe34e_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054de5350391dc4305b1cc9148aa6fe34e_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,479 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ced2db782e6425891db1f38a331c626_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412057ced2db782e6425891db1f38a331c626_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,480 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120580dd03c6319e4d858b180310f5e9539c_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120580dd03c6319e4d858b180310f5e9539c_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,480 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059108ea12e64c44348226e51d38907f07_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059108ea12e64c44348226e51d38907f07_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,481 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a22af6599a2c46049ce62ebb51deea47_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a22af6599a2c46049ce62ebb51deea47_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,482 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ce8080f07724451fb349f95959de042b_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ce8080f07724451fb349f95959de042b_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,482 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d0864854835342faab8fbf540c4f62b7_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d0864854835342faab8fbf540c4f62b7_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,483 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ed0e7ab0f0f24a0caf04914fd45eaad6_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ed0e7ab0f0f24a0caf04914fd45eaad6_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,484 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ef12ca26446a4c4d82a53887c3c5888c_e2b2ca43411f4bcc833039c57163fe52 to hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ef12ca26446a4c4d82a53887c3c5888c_e2b2ca43411f4bcc833039c57163fe52 2024-12-05T07:42:59,484 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T07:42:59,486 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=160, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:59,487 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T07:42:59,489 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T07:42:59,490 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=160, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:59,490 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T07:42:59,490 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733384579490"}]},"ts":"9223372036854775807"} 2024-12-05T07:42:59,491 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T07:42:59,491 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e2b2ca43411f4bcc833039c57163fe52, NAME => 'TestAcidGuarantees,,1733384549133.e2b2ca43411f4bcc833039c57163fe52.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T07:42:59,491 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T07:42:59,491 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733384579491"}]},"ts":"9223372036854775807"} 2024-12-05T07:42:59,492 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T07:42:59,493 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=160, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T07:42:59,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-12-05T07:42:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33643 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T07:42:59,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T07:42:59,567 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T07:42:59,580 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239 (was 236) - Thread LEAK? -, OpenFileDescriptor=451 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=361 (was 356) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6281 (was 6409) 2024-12-05T07:42:59,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T07:42:59,580 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T07:42:59,580 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T07:42:59,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:59,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:59,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T07:42:59,580 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T07:42:59,580 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-12-05T07:42:59,581 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fc6cd38557f3,33643,1733384393481 2024-12-05T07:42:59,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T07:42:59,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T07:42:59,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:42:59,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:42:59,616 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T07:42:59,617 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T07:42:59,617 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T07:42:59,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:59,618 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T07:42:59,619 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fc6cd38557f3,36693,1733384394401' ***** 2024-12-05T07:42:59,619 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T07:42:59,619 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T07:42:59,620 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T07:42:59,621 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T07:42:59,621 INFO [RS:0;fc6cd38557f3:36693 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T07:42:59,621 INFO [RS:0;fc6cd38557f3:36693 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T07:42:59,622 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(959): stopping server fc6cd38557f3,36693,1733384394401 2024-12-05T07:42:59,622 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T07:42:59,622 INFO [RS:0;fc6cd38557f3:36693 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fc6cd38557f3:36693. 2024-12-05T07:42:59,622 DEBUG [RS:0;fc6cd38557f3:36693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T07:42:59,622 DEBUG [RS:0;fc6cd38557f3:36693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:42:59,623 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T07:42:59,623 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T07:42:59,623 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T07:42:59,623 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T07:42:59,623 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T07:42:59,624 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-05T07:42:59,624 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T07:42:59,624 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T07:42:59,624 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T07:42:59,624 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T07:42:59,624 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T07:42:59,624 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T07:42:59,624 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-05T07:42:59,642 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/info/f14243ad16ef4c0f8dbe43a1fe66461d is 95, key is TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5./info:/1733384422864/DeleteFamily/seqid=0 2024-12-05T07:42:59,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742401_1577 (size=6514) 2024-12-05T07:42:59,648 INFO [regionserver/fc6cd38557f3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T07:42:59,648 INFO [regionserver/fc6cd38557f3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T07:42:59,666 INFO [regionserver/fc6cd38557f3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T07:42:59,824 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T07:43:00,025 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T07:43:00,046 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/info/f14243ad16ef4c0f8dbe43a1fe66461d 2024-12-05T07:43:00,068 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/ns/ab1bdc93f132456cafbbd64382273837 is 93, key is TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5./ns:/1733384422818/DeleteFamily/seqid=0 2024-12-05T07:43:00,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742402_1578 (size=5906) 2024-12-05T07:43:00,225 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T07:43:00,425 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T07:43:00,473 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/ns/ab1bdc93f132456cafbbd64382273837 2024-12-05T07:43:00,498 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/rep_barrier/d7e9ae4d08aa497bb9e5fb14bc2a00af is 102, key is TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5./rep_barrier:/1733384422818/DeleteFamily/seqid=0 2024-12-05T07:43:00,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742403_1579 (size=6025) 2024-12-05T07:43:00,626 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T07:43:00,826 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T07:43:00,901 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/rep_barrier/d7e9ae4d08aa497bb9e5fb14bc2a00af 2024-12-05T07:43:00,919 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/table/4f88834c47564c308b5c037ac382d230 is 96, key is TestAcidGuarantees,,1733384397759.27ae0e4955183fa53f189012e681dac5./table:/1733384422818/DeleteFamily/seqid=0 2024-12-05T07:43:00,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742404_1580 (size=5936) 2024-12-05T07:43:01,026 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T07:43:01,226 DEBUG [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T07:43:01,323 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/table/4f88834c47564c308b5c037ac382d230 2024-12-05T07:43:01,331 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/info/f14243ad16ef4c0f8dbe43a1fe66461d as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/info/f14243ad16ef4c0f8dbe43a1fe66461d 2024-12-05T07:43:01,338 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/info/f14243ad16ef4c0f8dbe43a1fe66461d, entries=12, sequenceid=90, filesize=6.4 K 2024-12-05T07:43:01,339 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/ns/ab1bdc93f132456cafbbd64382273837 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/ns/ab1bdc93f132456cafbbd64382273837 2024-12-05T07:43:01,342 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/ns/ab1bdc93f132456cafbbd64382273837, entries=8, sequenceid=90, filesize=5.8 K 2024-12-05T07:43:01,343 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/rep_barrier/d7e9ae4d08aa497bb9e5fb14bc2a00af as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/rep_barrier/d7e9ae4d08aa497bb9e5fb14bc2a00af 2024-12-05T07:43:01,347 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/rep_barrier/d7e9ae4d08aa497bb9e5fb14bc2a00af, entries=6, sequenceid=90, filesize=5.9 K 2024-12-05T07:43:01,348 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/.tmp/table/4f88834c47564c308b5c037ac382d230 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/table/4f88834c47564c308b5c037ac382d230 2024-12-05T07:43:01,352 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/table/4f88834c47564c308b5c037ac382d230, entries=7, sequenceid=90, filesize=5.8 K 2024-12-05T07:43:01,353 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1729ms, sequenceid=90, compaction requested=false 2024-12-05T07:43:01,357 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-05T07:43:01,358 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T07:43:01,358 INFO [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T07:43:01,358 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733384579623Running coprocessor pre-close hooks at 1733384579624 (+1 ms)Disabling compacts and flushes for region at 1733384579624Disabling writes for close at 1733384579624Obtaining lock to block concurrent updates at 1733384579624Preparing flush snapshotting stores in 1588230740 at 1733384579624Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733384579624Flushing stores of hbase:meta,,1.1588230740 at 1733384579625 (+1 ms)Flushing 1588230740/info: creating writer at 1733384579625Flushing 1588230740/info: appending metadata at 1733384579642 (+17 ms)Flushing 1588230740/info: closing flushed file at 1733384579642Flushing 1588230740/ns: creating writer at 1733384580054 (+412 ms)Flushing 1588230740/ns: appending metadata at 1733384580068 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733384580068Flushing 1588230740/rep_barrier: creating writer at 1733384580479 (+411 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733384580497 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733384580497Flushing 1588230740/table: creating writer at 1733384580905 (+408 ms)Flushing 1588230740/table: appending metadata at 1733384580919 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733384580919Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@378fb75f: reopening flushed file at 1733384581330 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35b5ebd7: reopening flushed file at 1733384581338 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@568c8e88: reopening flushed file at 1733384581343 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69809611: reopening flushed file at 1733384581348 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1729ms, sequenceid=90, compaction requested=false at 1733384581353 (+5 ms)Writing region close event to WAL at 1733384581354 (+1 ms)Running coprocessor post-close hooks at 1733384581358 (+4 ms)Closed at 1733384581358 2024-12-05T07:43:01,359 DEBUG [RS_CLOSE_META-regionserver/fc6cd38557f3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T07:43:01,427 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(976): stopping server fc6cd38557f3,36693,1733384394401; all regions closed. 2024-12-05T07:43:01,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741834_1010 (size=25344) 2024-12-05T07:43:01,441 DEBUG [RS:0;fc6cd38557f3:36693 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/oldWALs 2024-12-05T07:43:01,441 INFO [RS:0;fc6cd38557f3:36693 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fc6cd38557f3%2C36693%2C1733384394401.meta:.meta(num 1733384397318) 2024-12-05T07:43:01,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741833_1009 (size=13246867) 2024-12-05T07:43:01,446 DEBUG [RS:0;fc6cd38557f3:36693 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/oldWALs 2024-12-05T07:43:01,446 INFO [RS:0;fc6cd38557f3:36693 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fc6cd38557f3%2C36693%2C1733384394401:(num 1733384396719) 2024-12-05T07:43:01,446 DEBUG [RS:0;fc6cd38557f3:36693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T07:43:01,446 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T07:43:01,447 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T07:43:01,447 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.ChoreService(370): Chore service for: regionserver/fc6cd38557f3:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T07:43:01,447 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T07:43:01,447 INFO [regionserver/fc6cd38557f3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T07:43:01,448 INFO [RS:0;fc6cd38557f3:36693 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36693 2024-12-05T07:43:01,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T07:43:01,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fc6cd38557f3,36693,1733384394401 2024-12-05T07:43:01,524 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T07:43:01,545 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fc6cd38557f3,36693,1733384394401] 2024-12-05T07:43:01,557 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fc6cd38557f3,36693,1733384394401 already deleted, retry=false 2024-12-05T07:43:01,557 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fc6cd38557f3,36693,1733384394401 expired; onlineServers=0 2024-12-05T07:43:01,558 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fc6cd38557f3,33643,1733384393481' ***** 2024-12-05T07:43:01,558 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T07:43:01,558 INFO [M:0;fc6cd38557f3:33643 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T07:43:01,558 INFO [M:0;fc6cd38557f3:33643 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T07:43:01,558 DEBUG [M:0;fc6cd38557f3:33643 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T07:43:01,558 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T07:43:01,558 DEBUG [M:0;fc6cd38557f3:33643 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T07:43:01,558 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster-HFileCleaner.large.0-1733384396405 {}] cleaner.HFileCleaner(306): Exit Thread[master/fc6cd38557f3:0:becomeActiveMaster-HFileCleaner.large.0-1733384396405,5,FailOnTimeoutGroup] 2024-12-05T07:43:01,558 DEBUG [master/fc6cd38557f3:0:becomeActiveMaster-HFileCleaner.small.0-1733384396407 {}] cleaner.HFileCleaner(306): Exit Thread[master/fc6cd38557f3:0:becomeActiveMaster-HFileCleaner.small.0-1733384396407,5,FailOnTimeoutGroup] 2024-12-05T07:43:01,559 INFO [M:0;fc6cd38557f3:33643 {}] hbase.ChoreService(370): Chore service for: master/fc6cd38557f3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T07:43:01,559 INFO [M:0;fc6cd38557f3:33643 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T07:43:01,559 DEBUG [M:0;fc6cd38557f3:33643 {}] master.HMaster(1795): Stopping service threads 2024-12-05T07:43:01,559 INFO [M:0;fc6cd38557f3:33643 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T07:43:01,560 INFO [M:0;fc6cd38557f3:33643 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T07:43:01,560 ERROR [M:0;fc6cd38557f3:33643 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-05T07:43:01,561 INFO [M:0;fc6cd38557f3:33643 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T07:43:01,561 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T07:43:01,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T07:43:01,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T07:43:01,569 DEBUG [M:0;fc6cd38557f3:33643 {}] zookeeper.ZKUtil(347): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T07:43:01,569 WARN [M:0;fc6cd38557f3:33643 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T07:43:01,570 INFO [M:0;fc6cd38557f3:33643 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/.lastflushedseqids 2024-12-05T07:43:01,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742405_1581 (size=72) 2024-12-05T07:43:01,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T07:43:01,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36693-0x101a5b862740001, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T07:43:01,646 INFO [RS:0;fc6cd38557f3:36693 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T07:43:01,646 INFO [RS:0;fc6cd38557f3:36693 {}] regionserver.HRegionServer(1031): Exiting; stopping=fc6cd38557f3,36693,1733384394401; zookeeper connection closed. 2024-12-05T07:43:01,647 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@32b666f6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@32b666f6 2024-12-05T07:43:01,648 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T07:43:01,979 INFO [M:0;fc6cd38557f3:33643 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T07:43:01,980 INFO [M:0;fc6cd38557f3:33643 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T07:43:01,980 DEBUG [M:0;fc6cd38557f3:33643 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T07:43:01,980 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T07:43:01,980 DEBUG [M:0;fc6cd38557f3:33643 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T07:43:01,981 DEBUG [M:0;fc6cd38557f3:33643 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T07:43:01,981 DEBUG [M:0;fc6cd38557f3:33643 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T07:43:01,981 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=722.97 KB heapSize=886.76 KB 2024-12-05T07:43:02,000 DEBUG [M:0;fc6cd38557f3:33643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e1d47d69d126460bb7a8f22cd1fba555 is 82, key is hbase:meta,,1/info:regioninfo/1733384397392/Put/seqid=0 2024-12-05T07:43:02,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742406_1582 (size=5672) 2024-12-05T07:43:02,404 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2017 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e1d47d69d126460bb7a8f22cd1fba555 2024-12-05T07:43:02,430 DEBUG [M:0;fc6cd38557f3:33643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9177bdc331434e31af66188e8ace4243 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x89/proc:d/1733384552238/Put/seqid=0 2024-12-05T07:43:02,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742407_1583 (size=42389) 2024-12-05T07:43:02,835 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=722.41 KB at sequenceid=2017 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9177bdc331434e31af66188e8ace4243 2024-12-05T07:43:02,842 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9177bdc331434e31af66188e8ace4243 2024-12-05T07:43:02,860 DEBUG [M:0;fc6cd38557f3:33643 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/095459688fcd4bd483d015baaf9c42cc is 69, key is fc6cd38557f3,36693,1733384394401/rs:state/1733384396462/Put/seqid=0 2024-12-05T07:43:02,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073742408_1584 (size=5156) 2024-12-05T07:43:03,265 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2017 (bloomFilter=true), to=hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/095459688fcd4bd483d015baaf9c42cc 2024-12-05T07:43:03,275 DEBUG [M:0;fc6cd38557f3:33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e1d47d69d126460bb7a8f22cd1fba555 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e1d47d69d126460bb7a8f22cd1fba555 2024-12-05T07:43:03,280 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e1d47d69d126460bb7a8f22cd1fba555, entries=8, sequenceid=2017, filesize=5.5 K 2024-12-05T07:43:03,281 DEBUG [M:0;fc6cd38557f3:33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9177bdc331434e31af66188e8ace4243 as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9177bdc331434e31af66188e8ace4243 2024-12-05T07:43:03,286 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9177bdc331434e31af66188e8ace4243 2024-12-05T07:43:03,286 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9177bdc331434e31af66188e8ace4243, entries=160, sequenceid=2017, filesize=41.4 K 2024-12-05T07:43:03,287 DEBUG [M:0;fc6cd38557f3:33643 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/095459688fcd4bd483d015baaf9c42cc as hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/095459688fcd4bd483d015baaf9c42cc 2024-12-05T07:43:03,290 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36945/user/jenkins/test-data/480f4502-c2bb-755f-b4b9-2af038cacb15/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/095459688fcd4bd483d015baaf9c42cc, entries=1, sequenceid=2017, filesize=5.0 K 2024-12-05T07:43:03,290 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.HRegion(3140): Finished flush of dataSize ~722.97 KB/740320, heapSize ~886.46 KB/907736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1309ms, sequenceid=2017, compaction requested=false 2024-12-05T07:43:03,291 INFO [M:0;fc6cd38557f3:33643 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T07:43:03,291 DEBUG [M:0;fc6cd38557f3:33643 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733384581980Disabling compacts and flushes for region at 1733384581980Disabling writes for close at 1733384581981 (+1 ms)Obtaining lock to block concurrent updates at 1733384581981Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733384581981Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=740320, getHeapSize=907976, getOffHeapSize=0, getCellsCount=2202 at 1733384581982 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733384581983 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733384581983Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733384581999 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733384581999Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733384582412 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733384582430 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733384582430Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733384582842 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733384582860 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733384582860Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c16f19a: reopening flushed file at 1733384583274 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d86a2e6: reopening flushed file at 1733384583280 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4539ed09: reopening flushed file at 1733384583286 (+6 ms)Finished flush of dataSize ~722.97 KB/740320, heapSize ~886.46 KB/907736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1309ms, sequenceid=2017, compaction requested=false at 1733384583290 (+4 ms)Writing region close event to WAL at 1733384583291 (+1 ms)Closed at 1733384583291 2024-12-05T07:43:03,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34247 is added to blk_1073741830_1006 (size=872584) 2024-12-05T07:43:03,294 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T07:43:03,294 INFO [M:0;fc6cd38557f3:33643 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T07:43:03,294 INFO [M:0;fc6cd38557f3:33643 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33643 2024-12-05T07:43:03,294 INFO [M:0;fc6cd38557f3:33643 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T07:43:03,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T07:43:03,445 INFO [M:0;fc6cd38557f3:33643 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T07:43:03,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33643-0x101a5b862740000, quorum=127.0.0.1:58664, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T07:43:03,455 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T07:43:03,457 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T07:43:03,457 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T07:43:03,457 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T07:43:03,457 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/hadoop.log.dir/,STOPPED} 2024-12-05T07:43:03,460 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T07:43:03,460 WARN [BP-332223701-172.17.0.2-1733384389686 heartbeating to localhost/127.0.0.1:36945 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T07:43:03,460 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T07:43:03,460 WARN [BP-332223701-172.17.0.2-1733384389686 heartbeating to localhost/127.0.0.1:36945 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-332223701-172.17.0.2-1733384389686 (Datanode Uuid 535bd6b0-421d-4e1c-a321-0e2166523bb2) service to localhost/127.0.0.1:36945 2024-12-05T07:43:03,462 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/cluster_8fe0b83e-62de-a3e8-93ab-b871591f10b7/data/data1/current/BP-332223701-172.17.0.2-1733384389686 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T07:43:03,462 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/cluster_8fe0b83e-62de-a3e8-93ab-b871591f10b7/data/data2/current/BP-332223701-172.17.0.2-1733384389686 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T07:43:03,463 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T07:43:03,469 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T07:43:03,470 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T07:43:03,470 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T07:43:03,470 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T07:43:03,470 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f72884fd-ba5c-ed2b-8945-71a7c84cf9f5/hadoop.log.dir/,STOPPED} 2024-12-05T07:43:03,483 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T07:43:03,590 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down